From 91ff77d20bc1e445eca314077fe77a3065657f8f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 17 Oct 2024 16:42:48 +0300 Subject: [PATCH 001/416] Git Check --- .../main/resources/sql/schema-entities.sql | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 9c95f385f8..c8872cdc15 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -1,5 +1,5 @@ -- --- Copyright © 2016-2024 The Thingsboard Authors +-- ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL -- -- Licensed under the Apache License, Version 2.0 (the "License"); -- you may not use this file except in compliance with the License. @@ -896,3 +896,31 @@ CREATE TABLE IF NOT EXISTS mobile_app_settings ( qr_code_config VARCHAR(100000), CONSTRAINT mobile_app_settings_tenant_id_unq_key UNIQUE (tenant_id) ); + +CREATE TABLE IF NOT EXISTS calculated_field ( + id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_id uuid NOT NULL, + type varchar(32) NOT NULL, + name varchar(255) NOT NULL, + configuration_version int DEFAULT 0, + configuration varchar(1000000), + version BIGINT DEFAULT 1, + external_id UUID, + CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), + CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id) +); + +CREATE TABLE IF NOT EXISTS calculated_field_link ( + id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_id uuid NOT NULL, +-- target_id uuid NOT NULL, + calculated_field_id uuid NOT NULL, + configuration varchar(10000), + CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), + CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id), + CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +); From fd0af733c1eba5c65fcedf4c4225d87f1983cdc3 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 17 Oct 2024 16:59:08 +0300 Subject: [PATCH 002/416] LICENSE CHECK --- LICENSE | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE b/LICENSE index c8f142f0bf..2717e8bf56 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ +12312321 CONFIDENTIAL Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ From d8fd9234def7fa2dba80d9328e53f887e6aa952d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 30 Oct 2024 17:16:39 +0200 Subject: [PATCH 003/416] Revert "LICENSE CHECK" This reverts commit fd0af733c1eba5c65fcedf4c4225d87f1983cdc3. --- LICENSE | 1 - 1 file changed, 1 deletion(-) diff --git a/LICENSE b/LICENSE index 2717e8bf56..c8f142f0bf 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,3 @@ -12312321 CONFIDENTIAL Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ From 32bbd835419b00b2e4dd02e973b13d3b00291e53 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 31 Oct 2024 16:42:28 +0200 Subject: [PATCH 004/416] added calculated field entity, dao and controller --- .../server/controller/BaseController.java | 12 ++ .../controller/CalculatedFieldController.java | 95 ++++++++++++++ .../service/security/permission/Resource.java | 4 +- .../permission/TenantAdminPermissions.java | 1 + .../CalculatedFieldService.java | 31 +++++ .../server/common/data/EntityType.java | 3 +- .../calculated_field/CalculatedField.java | 94 ++++++++++++++ .../common/data/id/CalculatedFieldId.java | 44 +++++++ .../common/data/id/EntityIdFactory.java | 2 + common/proto/src/main/proto/queue.proto | 1 + .../BaseCalculatedFieldService.java | 75 +++++++++++ .../calculated_field/CalculatedFieldDao.java | 22 ++++ .../server/dao/model/ModelConstants.java | 14 +++ .../dao/model/sql/CalculatedFieldEntity.java | 118 ++++++++++++++++++ .../CalculatedFieldRepository.java | 24 ++++ .../JpaCalculatedFieldDao.java | 47 +++++++ .../main/resources/sql/schema-entities.sql | 47 ++++--- 17 files changed, 617 insertions(+), 17 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index da7d6243fc..10323f4fa0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -67,6 +67,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; @@ -77,6 +78,7 @@ import org.thingsboard.server.common.data.id.AlarmCommentId; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -124,6 +126,7 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; @@ -343,6 +346,9 @@ public abstract class BaseController { @Autowired protected TbServiceInfoProvider serviceInfoProvider; + @Autowired + protected CalculatedFieldService calculatedFieldService; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -645,6 +651,8 @@ public abstract class BaseController { case MOBILE_APP: checkMobileAppId(new MobileAppId(entityId.getId()), operation); return; + case CALCULATED_FIELD: + checkCalculatedFieldId(new CalculatedFieldId(entityId.getId()), operation); default: checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); } @@ -915,6 +923,10 @@ public abstract class BaseController { } } + protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { + return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation); + } + protected MediaType parseMediaType(String contentType) { try { return MediaType.parseMediaType(contentType); diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java new file mode 100644 index 0000000000..6bb5fc15f9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import io.swagger.v3.oas.annotations.Parameter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.config.annotations.ApiOperation; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; + +import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +@RequiredArgsConstructor +@Slf4j +public class CalculatedFieldController extends BaseController{ + + private final CalculatedFieldService calculatedFieldService; + + public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; + + @ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", + notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + + "The newly created Calculated Field Id will be present in the response. " + + "Specify existing Calculated Field Id to update the calculated field. " + + "Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + + "Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " + + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) + @ResponseBody + public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") + @RequestBody CalculatedField calculatedField) throws Exception { + calculatedField.setTenantId(getTenantId()); + checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); + return calculatedFieldService.save(calculatedField); + } + + @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", + notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) + @ResponseBody + public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { + checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); + return checkCalculatedFieldId(calculatedFieldId, Operation.READ); + } + + + @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", + notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedField) throws Exception { + checkParameter(CALCULATED_FIELD_ID, strCalculatedField); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedField)); + checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); + calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 16a3c4be59..7ec040ecd8 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -49,7 +49,9 @@ public enum Resource { VERSION_CONTROL, NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), - MOBILE_APP_SETTINGS; + MOBILE_APP_SETTINGS, + CALCULATED_FIELD(EntityType.CALCULATED_FIELD); + private final Set entityTypes; Resource() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 10807e4b5a..897581cc91 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -51,6 +51,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.VERSION_CONTROL, PermissionChecker.allowAllPermissionChecker); put(Resource.NOTIFICATION, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_SETTINGS, new PermissionChecker.GenericPermissionChecker(Operation.READ)); + put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java new file mode 100644 index 0000000000..7f79f481b0 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entity.EntityDaoService; + +public interface CalculatedFieldService extends EntityDaoService { + + CalculatedField save(CalculatedField calculatedField); + + CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index b5ce79e20f..7b6e99095f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -60,7 +60,8 @@ public enum EntityType { QUEUE_STATS(34), OAUTH2_CLIENT(35), DOMAIN(36), - MOBILE_APP(37); + MOBILE_APP(37), + CALCULATED_FIELD(38); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java new file mode 100644 index 0000000000..d780262726 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.calculated_field; + +import com.fasterxml.jackson.databind.JsonNode; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.validation.Length; +import org.thingsboard.server.common.data.validation.NoXss; + +@Schema +@Data +@EqualsAndHashCode(callSuper = true) +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion { + + private static final long serialVersionUID = 4491966747773381420L; + + private TenantId tenantId; + private EntityId entityId; + + @NoXss + @Length(fieldName = "type") + private String type; + @NoXss + @Length(fieldName = "name") + @Schema(description = "User defined name of the calculated field.") + private String name; + @Schema(description = "Version of calculated field configuration.", example = "0") + private int configurationVersion; + @Schema(description = "JSON with the calculated field configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) + private transient JsonNode configuration; + @Getter + @Setter + private Long version; + @Getter + @Setter + private CalculatedFieldId externalId; + + public CalculatedField() { + super(); + } + + public CalculatedField(CalculatedFieldId id) { + super(id); + } + + public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, JsonNode configuration, Long version, CalculatedFieldId externalId) { + super(); + this.tenantId = tenantId; + this.entityId = entityId; + this.type = type; + this.name = name; + this.configurationVersion = configurationVersion; + this.configuration = configuration; + this.version = version; + this.externalId = externalId; + } + + @Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.") + @Override + public CalculatedFieldId getId() { + return super.getId(); + } + + @Schema(description = "Timestamp of the calculated field creation, in milliseconds", example = "1609459200000", accessMode = Schema.AccessMode.READ_ONLY) + @Override + public long getCreatedTime() { + return super.getCreatedTime(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java new file mode 100644 index 0000000000..0de6209b9d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Schema +public class CalculatedFieldId extends UUIDBased implements EntityId { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public CalculatedFieldId(@JsonProperty("id") UUID id) { + super(id); + } + + public static CalculatedFieldId fromString(String calculatedFieldId) { + return new CalculatedFieldId(UUID.fromString(calculatedFieldId)); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "CALCULATED_FIELD", allowableValues = "CALCULATED_FIELD") + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 5a85e6ce67..2b53011bdd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -111,6 +111,8 @@ public class EntityIdFactory { return new MobileAppId(uuid); case DOMAIN: return new DomainId(uuid); + case CALCULATED_FIELD: + return new CalculatedFieldId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index f84c7f7c9b..985e7518f5 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -58,6 +58,7 @@ enum EntityTypeProto { OAUTH2_CLIENT = 35; DOMAIN = 36; MOBILE_APP = 37; + CALCULATED_FIELD = 38; } /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java new file mode 100644 index 0000000000..f9f7681a3a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Optional; + +import static org.thingsboard.server.dao.service.Validator.validateId; + +@Service("CalculatedFieldDaoService") +@Slf4j +@RequiredArgsConstructor +public class BaseCalculatedFieldService implements CalculatedFieldService { + + public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; + public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; + + private final CalculatedFieldDao calculatedFieldDao; + + + @Override + public CalculatedField save(CalculatedField calculatedField) { + log.trace("Executing save, [{}]", calculatedField); + return calculatedFieldDao.save(calculatedField.getTenantId(), calculatedField); + } + + @Override + public CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findById, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); + } + + @Override + public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing deleteRpc, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); + } + + @Override + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); + } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java new file mode 100644 index 0000000000..71decef684 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.dao.Dao; + +public interface CalculatedFieldDao extends Dao { +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index f85cd4bac4..c37cbfca8e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -691,6 +691,20 @@ public class ModelConstants { public static final String MOBILE_APP_SETTINGS_IOS_CONFIG_PROPERTY = "ios_config"; public static final String MOBILE_APP_SETTINGS_QR_CODE_CONFIG_PROPERTY = "qr_code_config"; + /** + * Calculated fields constants. + */ + public static final String CALCULATED_FIELD_TABLE_NAME = "calculated_field"; + public static final String CALCULATED_FIELD_TENANT_ID_COLUMN = TENANT_ID_COLUMN; + public static final String CALCULATED_FIELD_ENTITY_TYPE = ENTITY_TYPE_COLUMN; + public static final String CALCULATED_FIELD_ENTITY_ID = ENTITY_ID_COLUMN; + public static final String CALCULATED_FIELD_TYPE = "type"; + public static final String CALCULATED_FIELD_NAME = "name"; + public static final String CALCULATED_FIELD_CONFIGURATION_VERSION = "configuration_version"; + public static final String CALCULATED_FIELD_CONFIGURATION = "configuration"; + public static final String CALCULATED_FIELD_VERSION = "version"; + public static final String CALCULATED_FIELD_EXTERNAL_ID = "external_id"; + protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java new file mode 100644 index 0000000000..35c50c561f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.util.mapping.JsonConverter; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_EXTERNAL_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TENANT_ID_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_VERSION; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_TABLE_NAME) +public class CalculatedFieldEntity extends BaseSqlEntity implements BaseEntity { + + @Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = CALCULATED_FIELD_ENTITY_TYPE) + private EntityType entityType; + + @Column(name = CALCULATED_FIELD_ENTITY_ID) + private UUID entityId; + + @Column(name = CALCULATED_FIELD_TYPE) + private String type; + + @Column(name = CALCULATED_FIELD_NAME) + private String name; + + @Column(name = CALCULATED_FIELD_CONFIGURATION_VERSION) + private int configurationVersion; + + @Convert(converter = JsonConverter.class) + @Column(name = CALCULATED_FIELD_CONFIGURATION) + private JsonNode configuration; + + @Column(name = CALCULATED_FIELD_VERSION) + private Long version; + + @Column(name = CALCULATED_FIELD_EXTERNAL_ID) + private UUID externalId; + + public CalculatedFieldEntity() { + super(); + } + + public CalculatedFieldEntity(CalculatedField calculatedField) { + this.setUuid(calculatedField.getUuidId()); + this.createdTime = calculatedField.getCreatedTime(); + this.tenantId = calculatedField.getTenantId().getId(); + this.entityType = calculatedField.getEntityId().getEntityType(); + this.entityId = calculatedField.getEntityId().getId(); + this.type = calculatedField.getType(); + this.name = calculatedField.getName(); + this.configurationVersion = calculatedField.getConfigurationVersion(); + this.configuration = calculatedField.getConfiguration(); + this.version = calculatedField.getVersion(); + if (calculatedField.getExternalId() != null) { + this.externalId = calculatedField.getExternalId().getId(); + } + } + + @Override + public CalculatedField toData() { + CalculatedField calculatedField = new CalculatedField(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(TenantId.fromUUID(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(type); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(configuration); + calculatedField.setVersion(version); + if (externalId != null) { + calculatedField.setExternalId(new CalculatedFieldId(externalId)); + } + return calculatedField; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java new file mode 100644 index 0000000000..79146cf317 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; + +import java.util.UUID; + +public interface CalculatedFieldRepository extends JpaRepository { +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java new file mode 100644 index 0000000000..33cf926eaf --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.UUID; + +@Slf4j +@Component +@AllArgsConstructor +@SqlDao +public class JpaCalculatedFieldDao extends JpaAbstractDao implements CalculatedFieldDao { + + private final CalculatedFieldRepository calculatedFieldRepository; + + @Override + protected Class getEntityClass() { + return CalculatedFieldEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return calculatedFieldRepository; + } +} diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a26bb03e62..545486180d 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -1,3 +1,19 @@ +-- +-- Copyright © 2016-2024 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + -- -- ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL -- @@ -891,6 +907,7 @@ CREATE TABLE IF NOT EXISTS calculated_field ( id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, created_time bigint NOT NULL, tenant_id uuid NOT NULL, + entity_type VARCHAR(32), entity_id uuid NOT NULL, type varchar(32) NOT NULL, name varchar(255) NOT NULL, @@ -899,18 +916,18 @@ CREATE TABLE IF NOT EXISTS calculated_field ( version BIGINT DEFAULT 1, external_id UUID, CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), - CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id) -); - -CREATE TABLE IF NOT EXISTS calculated_field_link ( - id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid NOT NULL, - entity_id uuid NOT NULL, --- target_id uuid NOT NULL, - calculated_field_id uuid NOT NULL, - configuration varchar(10000), - CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), - CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id), - CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE -); + CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) +); + +-- CREATE TABLE IF NOT EXISTS calculated_field_link ( +-- id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, +-- created_time bigint NOT NULL, +-- tenant_id uuid NOT NULL, +-- entity_id uuid NOT NULL, +-- -- target_id uuid NOT NULL, +-- calculated_field_id uuid NOT NULL, +-- configuration varchar(10000), +-- CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), +-- CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id), +-- CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +-- ); From e2d17a822d7519e0d611d06b1ec1abd2160e812c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 1 Nov 2024 13:34:04 +0200 Subject: [PATCH 005/416] added tests --- .../server/controller/BaseController.java | 1 + .../CalculatedFieldControllerTest.java | 143 ++++++++++++++++++ .../calculated_field/CalculatedField.java | 24 ++- .../BaseCalculatedFieldService.java | 1 - .../CalculatedFieldServiceTest.java | 123 +++++++++++++++ 5 files changed, 286 insertions(+), 6 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 10323f4fa0..6d52bdb358 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -653,6 +653,7 @@ public abstract class BaseController { return; case CALCULATED_FIELD: checkCalculatedFieldId(new CalculatedFieldId(entityId.getId()), operation); + return; default: checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java new file mode 100644 index 0000000000..05ffd03970 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -0,0 +1,143 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +public class CalculatedFieldControllerTest extends AbstractControllerTest { + + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("9e408b94-dc05-47e2-a21c-1a6c0d7bd90a")); + + private Tenant savedTenant; + private User tenantAdmin; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + savedTenant = saveTenant(tenant); + assertThat(savedTenant).isNotNull(); + + tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setFirstName("Joe"); + tenantAdmin.setLastName("Downs"); + + tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + deleteTenant(savedTenant.getId()); + } + + @Test + public void testSaveCalculatedField() throws Exception { + CalculatedField calculatedField = getCalculatedField(); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(savedTenant.getId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(calculatedField.getVersion()); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + assertThat(updatedCalculatedField).isEqualTo(savedCalculatedField); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testGetCalculatedFieldById() throws Exception { + CalculatedField calculatedField = getCalculatedField(); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + CalculatedField fetchedCalculatedField = doGet("/api/calculatedField/" + savedCalculatedField.getId().getId(), CalculatedField.class); + + assertThat(fetchedCalculatedField).isNotNull(); + assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testDeleteCalculatedField() throws Exception { + CalculatedField calculatedField = getCalculatedField(); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); + + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType("Simple"); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + + " \"T\": {\n" + + " \"key\": \"temperature\",\n" + + " \"type\": \"TIME_SERIES\"\n" + + " },\n" + + " \"H\": {\n" + + " \"key\": \"humidity\",\n" + + " \"type\": \"TIME_SERIES\",\n" + + " \"defaultValue\": 50\n" + + " }\n" + + " }\n")); + calculatedField.setVersion(1L); + return calculatedField; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java index d780262726..0bd8ded8f4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java @@ -21,10 +21,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,7 +31,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @Schema @Data @EqualsAndHashCode(callSuper = true) -public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion { +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity { private static final long serialVersionUID = 4491966747773381420L; @@ -91,4 +88,21 @@ public class CalculatedField extends BaseData implements HasN return super.getCreatedTime(); } + @Override + public String toString() { + return new StringBuilder() + .append("CalculatedField[") + .append("tenantId=").append(tenantId) + .append(", entityId=").append(entityId) + .append(", type='").append(type) + .append(", name='").append(name) + .append(", configurationVersion=").append(configurationVersion) + .append(", configuration=").append(configuration) + .append(", version=").append(version) + .append(", externalId=").append(externalId) + .append(", createdTime=").append(createdTime) + .append(", id=").append(id).append(']') + .toString(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java index f9f7681a3a..530a533085 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java @@ -39,7 +39,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { private final CalculatedFieldDao calculatedFieldDao; - @Override public CalculatedField save(CalculatedField calculatedField) { log.trace("Executing save, [{}]", calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java new file mode 100644 index 0000000000..d2e3d40c5c --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java @@ -0,0 +1,123 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.dao.service.AbstractServiceTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@DaoSqlTest +public class CalculatedFieldServiceTest extends AbstractServiceTest { + + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("71c73816-361e-4e57-82ab-e1deaa8b7d66")); + + @Autowired + private CalculatedFieldService calculatedFieldService; + + private ListeningExecutorService executor; + + @Before + public void before() { + executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(8, getClass())); + } + + @After + public void after() { + executor.shutdownNow(); + } + + @Test + public void testSaveCalculatedField() { + CalculatedField calculatedField = getCalculatedField(); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(calculatedField.getTenantId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(calculatedField.getVersion()); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = calculatedFieldService.save(savedCalculatedField); + + assertThat(updatedCalculatedField).isEqualTo(savedCalculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + + @Test + public void testFindCalculatedFieldById() { + CalculatedField calculatedField = getCalculatedField(); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + CalculatedField fetchedCalculatedField = calculatedFieldService.findById(tenantId, savedCalculatedField.getId()); + + assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + + @Test + public void testDeleteCalculatedField() { + CalculatedField calculatedField = getCalculatedField(); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + + assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull(); + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType("Simple"); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + + " \"T\": {\n" + + " \"key\": \"temperature\",\n" + + " \"type\": \"TIME_SERIES\"\n" + + " },\n" + + " \"H\": {\n" + + " \"key\": \"humidity\",\n" + + " \"type\": \"TIME_SERIES\",\n" + + " \"defaultValue\": 50\n" + + " }\n" + + " }\n")); + calculatedField.setVersion(1L); + return calculatedField; + } + +} From 03ff7c17ac0b4e7049f9bb7fc060e0a219237a9f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 4 Nov 2024 15:57:37 +0200 Subject: [PATCH 006/416] added calculated field link entity and its dao --- .../controller/CalculatedFieldController.java | 4 +- .../service/security/permission/Resource.java | 3 +- .../CalculatedFieldControllerTest.java | 23 +++-- .../CalculatedFieldService.java | 3 + .../server/common/data/EntityType.java | 3 +- .../calculated_field/CalculatedFieldLink.java | 71 ++++++++++++++ .../common/data/id/CalculatedFieldLinkId.java | 45 +++++++++ .../common/data/id/EntityIdFactory.java | 2 + common/proto/src/main/proto/queue.proto | 1 + .../BaseCalculatedFieldService.java | 66 ++++++++++++- .../calculated_field/CalculatedFieldDao.java | 2 +- .../CalculatedFieldLinkDao.java | 27 ++++++ .../server/dao/model/ModelConstants.java | 10 ++ .../model/sql/CalculatedFieldLinkEntity.java | 92 +++++++++++++++++++ .../CalculatedFieldDataValidator.java | 41 +++++++++ .../CalculatedFieldLinkDataValidator.java | 41 +++++++++ .../CalculatedFieldLinkRepository.java | 27 ++++++ .../JpaCalculatedFieldLinkDao.java | 54 +++++++++++ .../main/resources/sql/schema-entities.sql | 24 ++--- .../CalculatedFieldServiceTest.java | 90 ++++++++++++++++-- .../CalculatedFieldDataValidatorTest.java | 57 ++++++++++++ .../CalculatedFieldLinkDataValidatorTest.java | 57 ++++++++++++ 22 files changed, 702 insertions(+), 41 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 6bb5fc15f9..26a3539a46 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -44,7 +44,7 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI @RequestMapping("/api") @RequiredArgsConstructor @Slf4j -public class CalculatedFieldController extends BaseController{ +public class CalculatedFieldController extends BaseController { private final CalculatedFieldService calculatedFieldService; @@ -61,7 +61,7 @@ public class CalculatedFieldController extends BaseController{ @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) @ResponseBody public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") - @RequestBody CalculatedField calculatedField) throws Exception { + @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); return calculatedFieldService.save(calculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 7ec040ecd8..f57f460fc3 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -50,7 +50,8 @@ public enum Resource { NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), MOBILE_APP_SETTINGS, - CALCULATED_FIELD(EntityType.CALCULATED_FIELD); + CALCULATED_FIELD(EntityType.CALCULATED_FIELD), + CALCULATED_FIELD_LINK(EntityType.CALCULATED_FIELD_LINK); private final Set entityTypes; diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 05ffd03970..685a58d48c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -19,6 +19,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.calculated_field.CalculatedField; @@ -26,18 +27,13 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.service.DaoSqlTest; -import java.util.UUID; - import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest public class CalculatedFieldControllerTest extends AbstractControllerTest { - private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("9e408b94-dc05-47e2-a21c-1a6c0d7bd90a")); - private Tenant savedTenant; - private User tenantAdmin; @Before public void beforeTest() throws Exception { @@ -48,14 +44,14 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { savedTenant = saveTenant(tenant); assertThat(savedTenant).isNotNull(); - tenantAdmin = new User(); + User tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); tenantAdmin.setEmail("tenant2@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); - tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); + createUserAndLogin(tenantAdmin, "testPassword1"); } @After @@ -67,7 +63,8 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testSaveCalculatedField() throws Exception { - CalculatedField calculatedField = getCalculatedField(); + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); @@ -93,7 +90,8 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testGetCalculatedFieldById() throws Exception { - CalculatedField calculatedField = getCalculatedField(); + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); CalculatedField fetchedCalculatedField = doGet("/api/calculatedField/" + savedCalculatedField.getId().getId(), CalculatedField.class); @@ -107,7 +105,8 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testDeleteCalculatedField() throws Exception { - CalculatedField calculatedField = getCalculatedField(); + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); @@ -119,9 +118,9 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { } - private CalculatedField getCalculatedField() { + private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); - calculatedField.setEntityId(DEVICE_ID); + calculatedField.setEntityId(deviceId); calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java index 7f79f481b0..e1779e1d33 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; @@ -28,4 +29,6 @@ public interface CalculatedFieldService extends EntityDaoService { void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index 7b6e99095f..37d77bd2c0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -61,7 +61,8 @@ public enum EntityType { OAUTH2_CLIENT(35), DOMAIN(36), MOBILE_APP(37), - CALCULATED_FIELD(38); + CALCULATED_FIELD(38), + CALCULATED_FIELD_LINK(39); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java new file mode 100644 index 0000000000..79f05c9b58 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.calculated_field; + +import com.fasterxml.jackson.databind.JsonNode; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@Schema +@Data +@EqualsAndHashCode(callSuper = true) +public class CalculatedFieldLink extends BaseData { + + private static final long serialVersionUID = 6492846246722091530L; + + private TenantId tenantId; + private EntityId entityId; + + @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) + private CalculatedFieldId calculatedFieldId; + @Schema(description = "JSON with the calculated field link configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) + private transient JsonNode configuration; + + public CalculatedFieldLink() { + super(); + } + + public CalculatedFieldLink(CalculatedFieldLinkId id) { + super(id); + } + + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, JsonNode configuration, CalculatedFieldId calculatedFieldId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.configuration = configuration; + this.calculatedFieldId = calculatedFieldId; + } + + @Override + public String toString() { + return new StringBuilder() + .append("CalculatedFieldLink[") + .append("tenantId=").append(tenantId) + .append(", entityId=").append(entityId) + .append(", calculatedFieldId=").append(calculatedFieldId) + .append(", configuration=").append(configuration) + .append(", createdTime=").append(createdTime) + .append(", id=").append(id).append(']') + .toString(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java new file mode 100644 index 0000000000..8817aa8a1e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Schema +public class CalculatedFieldLinkId extends UUIDBased implements EntityId { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public CalculatedFieldLinkId(@JsonProperty("id") UUID id) { + super(id); + } + + public static CalculatedFieldLinkId fromString(String calculatedFieldLinkId) { + return new CalculatedFieldLinkId(UUID.fromString(calculatedFieldLinkId)); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "CALCULATED_FIELD_LINK", allowableValues = "CALCULATED_FIELD_LINK") + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD_LINK; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 2b53011bdd..973aa5202e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -113,6 +113,8 @@ public class EntityIdFactory { return new DomainId(uuid); case CALCULATED_FIELD: return new CalculatedFieldId(uuid); + case CALCULATED_FIELD_LINK: + return new CalculatedFieldLinkId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 985e7518f5..875a531566 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -59,6 +59,7 @@ enum EntityTypeProto { DOMAIN = 36; MOBILE_APP = 37; CALCULATED_FIELD = 38; + CALCULATED_FIELD_LINK = 39; } /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java index 530a533085..e7bc3be314 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java @@ -20,13 +20,21 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.service.DataValidator; +import java.util.Objects; import java.util.Optional; +import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; @Service("CalculatedFieldDaoService") @@ -38,11 +46,28 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; private final CalculatedFieldDao calculatedFieldDao; + private final CalculatedFieldLinkDao calculatedFieldLinkDao; + private final DeviceService deviceService; + private final AssetService assetService; + private final DataValidator calculatedFieldDataValidator; + private final DataValidator calculatedFieldLinkDataValidator; @Override public CalculatedField save(CalculatedField calculatedField) { - log.trace("Executing save, [{}]", calculatedField); - return calculatedFieldDao.save(calculatedField.getTenantId(), calculatedField); + calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); + try { + TenantId tenantId = calculatedField.getTenantId(); + checkEntityExistence(tenantId, calculatedField.getEntityId()); + log.trace("Executing save calculated field, [{}]", calculatedField); + CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); + createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + return savedCalculatedField; + } catch (Exception e) { + checkConstraintViolation(e, + "calculated_field_unq_key", "Calculated Field with such name is already in exists!", + "calculated_field_external_id_unq_key", "Calculated Field with such external id already exists!"); + throw e; + } } @Override @@ -61,6 +86,18 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); } + @Override + public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { + calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); + try { + log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); + return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); + } catch (Exception e) { + checkConstraintViolation(e, "calculated_field_link_unq_key", "Calculated Field for such entity id is already exists!"); + throw e; + } + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); @@ -71,4 +108,29 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return EntityType.CALCULATED_FIELD; } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { + switch (entityId.getEntityType()) { + case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); + case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); + default -> + throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' is not supported."); + } + } + + private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldLink calculatedFieldLink = calculatedFieldLinkDao.findCalculatedFieldLinkByEntityId(tenantId.getId(), calculatedField.getEntityId().getId()); + saveCalculatedFieldLink(tenantId, Objects.requireNonNullElseGet(calculatedFieldLink, () -> createCalculatedFieldLink(tenantId, calculatedField))); + } + + private CalculatedFieldLink createCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + calculatedFieldLink.setEntityId(calculatedField.getEntityId()); + calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); + calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); + return calculatedFieldLink; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java index 71decef684..e4bd019b18 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -18,5 +18,5 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.dao.Dao; -public interface CalculatedFieldDao extends Dao { +public interface CalculatedFieldDao extends Dao { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java new file mode 100644 index 0000000000..1c422311eb --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.dao.Dao; + +import java.util.UUID; + +public interface CalculatedFieldLinkDao extends Dao { + + CalculatedFieldLink findCalculatedFieldLinkByEntityId(UUID tenantId, UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index c37cbfca8e..368478db6b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -705,6 +705,16 @@ public class ModelConstants { public static final String CALCULATED_FIELD_VERSION = "version"; public static final String CALCULATED_FIELD_EXTERNAL_ID = "external_id"; + /** + * Calculated field links constants. + */ + public static final String CALCULATED_FIELD_LINK_TABLE_NAME = "calculated_field_link"; + public static final String CALCULATED_FIELD_LINK_TENANT_ID_COLUMN = TENANT_ID_COLUMN; + public static final String CALCULATED_FIELD_LINK_ENTITY_TYPE = ENTITY_TYPE_COLUMN; + public static final String CALCULATED_FIELD_LINK_ENTITY_ID = ENTITY_ID_COLUMN; + public static final String CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID = "calculated_field_id"; + public static final String CALCULATED_FIELD_LINK_CONFIGURATION = "configuration"; + protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java new file mode 100644 index 0000000000..9f2efb230b --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.util.mapping.JsonConverter; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TENANT_ID_COLUMN; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_LINK_TABLE_NAME) +public class CalculatedFieldLinkEntity extends BaseSqlEntity implements BaseEntity { + + @Column(name = CALCULATED_FIELD_LINK_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = CALCULATED_FIELD_LINK_ENTITY_TYPE) + private EntityType entityType; + + @Column(name = CALCULATED_FIELD_LINK_ENTITY_ID) + private UUID entityId; + + @Column(name = CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID) + private UUID calculatedFieldId; + + @Convert(converter = JsonConverter.class) + @Column(name = CALCULATED_FIELD_LINK_CONFIGURATION) + private JsonNode configuration; + + + public CalculatedFieldLinkEntity() { + super(); + } + + public CalculatedFieldLinkEntity(CalculatedFieldLink calculatedFieldLink) { + this.setUuid(calculatedFieldLink.getUuidId()); + this.createdTime = calculatedFieldLink.getCreatedTime(); + this.tenantId = calculatedFieldLink.getTenantId().getId(); + this.entityType = calculatedFieldLink.getEntityId().getEntityType(); + this.entityId = calculatedFieldLink.getEntityId().getId(); + this.calculatedFieldId = calculatedFieldLink.getCalculatedFieldId().getId(); + this.configuration = calculatedFieldLink.getConfiguration(); + } + + @Override + public CalculatedFieldLink toData() { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(TenantId.fromUUID(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + calculatedFieldLink.setConfiguration(configuration); + return calculatedFieldLink; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java new file mode 100644 index 0000000000..e50f2c902a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; + +@Component +public class CalculatedFieldDataValidator extends DataValidator { + + @Autowired + private CalculatedFieldDao calculatedFieldDao; + + @Override + protected CalculatedField validateUpdate(TenantId tenantId, CalculatedField calculatedField) { + CalculatedField old = calculatedFieldDao.findById(calculatedField.getTenantId(), calculatedField.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing calculated field!"); + } + return old; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java new file mode 100644 index 0000000000..83c30d4d03 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; + +@Component +public class CalculatedFieldLinkDataValidator extends DataValidator { + + @Autowired + private CalculatedFieldLinkDao calculatedFieldLinkDao; + + @Override + protected CalculatedFieldLink validateUpdate(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { + CalculatedFieldLink old = calculatedFieldLinkDao.findById(calculatedFieldLink.getTenantId(), calculatedFieldLink.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing calculated field link!"); + } + return old; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java new file mode 100644 index 0000000000..1339cf4478 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; + +import java.util.UUID; + +public interface CalculatedFieldLinkRepository extends JpaRepository { + + CalculatedFieldLinkEntity findByTenantIdAndEntityId(UUID tenantId, UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java new file mode 100644 index 0000000000..0721e08d7f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.UUID; + +@Slf4j +@Component +@AllArgsConstructor +@SqlDao +public class JpaCalculatedFieldLinkDao extends JpaAbstractDao implements CalculatedFieldLinkDao { + + private final CalculatedFieldLinkRepository calculatedFieldLinkRepository; + + @Override + public CalculatedFieldLink findCalculatedFieldLinkByEntityId(UUID tenantId, UUID entityId) { + return DaoUtil.getData(calculatedFieldLinkRepository.findByTenantIdAndEntityId(tenantId, entityId)); + } + + @Override + protected Class getEntityClass() { + return CalculatedFieldLinkEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return calculatedFieldLinkRepository; + } + +} diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 545486180d..790317fcb0 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -919,15 +919,15 @@ CREATE TABLE IF NOT EXISTS calculated_field ( CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) ); --- CREATE TABLE IF NOT EXISTS calculated_field_link ( --- id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, --- created_time bigint NOT NULL, --- tenant_id uuid NOT NULL, --- entity_id uuid NOT NULL, --- -- target_id uuid NOT NULL, --- calculated_field_id uuid NOT NULL, --- configuration varchar(10000), --- CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), --- CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id), --- CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE --- ); +CREATE TABLE IF NOT EXISTS calculated_field_link ( + id uuid NOT NULL CONSTRAINT calculated_field_link_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_type VARCHAR(32), + entity_id uuid NOT NULL, +-- target_id uuid NOT NULL, + calculated_field_id uuid NOT NULL, + configuration varchar(1000000), + CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), + CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +); diff --git a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java index d2e3d40c5c..fdff4bc2f6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java @@ -23,22 +23,28 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.AbstractServiceTest; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class CalculatedFieldServiceTest extends AbstractServiceTest { - private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("71c73816-361e-4e57-82ab-e1deaa8b7d66")); - @Autowired private CalculatedFieldService calculatedFieldService; + @Autowired + private DeviceService deviceService; private ListeningExecutorService executor; @@ -54,7 +60,8 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedField() { - CalculatedField calculatedField = getCalculatedField(); + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThat(savedCalculatedField).isNotNull(); @@ -77,10 +84,42 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { } @Test - public void testFindCalculatedFieldById() { - CalculatedField calculatedField = getCalculatedField(); - CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + public void testSaveCalculatesFieldWithNonExistingDeviceId() { + CalculatedField calculatedField = getCalculatedField(new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22"))); + + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Device with id [" + calculatedField.getEntityId().getId() + "] does not exist."); + } + + @Test + public void testSaveCalculatedFieldWithExistingName() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field with such name is already in exists!"); + } + + @Test + public void testSaveCalculatedFieldWithExistingExternalId() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); + calculatedField.setExternalId(new CalculatedFieldId(UUID.fromString("2ef69d0a-89cf-4868-86f8-c50551d87ebe"))); + + calculatedFieldService.save(calculatedField); + + calculatedField.setName("Test 2"); + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field with such external id already exists!"); + } + @Test + public void testFindCalculatedFieldById() { + CalculatedField savedCalculatedField = saveValidCalculatedField(); CalculatedField fetchedCalculatedField = calculatedFieldService.findById(tenantId, savedCalculatedField.getId()); assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); @@ -90,18 +129,33 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testDeleteCalculatedField() { - CalculatedField calculatedField = getCalculatedField(); - CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + CalculatedField savedCalculatedField = saveValidCalculatedField(); calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull(); } - private CalculatedField getCalculatedField() { + @Test + public void testSaveCalculatedFieldLinkIfCalculatedFieldForSuchEntityExists() { + CalculatedField savedCalculatedField = saveValidCalculatedField(); + CalculatedFieldLink calculatedFieldLink = getCalculatedFieldLink(savedCalculatedField); + + assertThatThrownBy(() -> calculatedFieldService.saveCalculatedFieldLink(tenantId, calculatedFieldLink)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field for such entity id is already exists!"); + } + + private CalculatedField saveValidCalculatedField() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); + return calculatedFieldService.save(calculatedField); + } + + private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); - calculatedField.setEntityId(DEVICE_ID); + calculatedField.setEntityId(deviceId); calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); @@ -120,4 +174,20 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { return calculatedField; } + private CalculatedFieldLink getCalculatedFieldLink(CalculatedField calculatedField) { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + calculatedFieldLink.setEntityId(calculatedField.getEntityId()); + calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); + calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); + return calculatedFieldLink; + } + + private Device createTestDevice() { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Test"); + return deviceService.saveDevice(device); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java new file mode 100644 index 0000000000..bc7990ca64 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; + +@SpringBootTest(classes = CalculatedFieldDataValidator.class) +public class CalculatedFieldDataValidatorTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("7b5229e9-166e-41a9-a257-3b1dafad1b04")); + private final CalculatedFieldId CALCULATED_FIELD_ID = new CalculatedFieldId(UUID.fromString("060fbe45-fbb2-4549-abf3-f72a6be3cb9f")); + + @MockBean + private CalculatedFieldDao calculatedFieldDao; + @SpyBean + private CalculatedFieldDataValidator validator; + + @Test + public void testUpdateNonExistingCalculatedField() { + CalculatedField calculatedField = new CalculatedField(CALCULATED_FIELD_ID); + calculatedField.setType("Simple"); + calculatedField.setName("Test"); + + given(calculatedFieldDao.findById(TENANT_ID, CALCULATED_FIELD_ID.getId())).willReturn(null); + + assertThatThrownBy(() -> validator.validateUpdate(TENANT_ID, calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't update non existing calculated field!"); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java new file mode 100644 index 0000000000..a74c59d7ad --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; + +@SpringBootTest(classes = CalculatedFieldLinkDataValidator.class) +public class CalculatedFieldLinkDataValidatorTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("2ba09d99-6143-43dc-b645-381fc0c43ebe")); + private final CalculatedFieldLinkId CALCULATED_FIELD_LINK_ID = new CalculatedFieldLinkId(UUID.fromString("a5609ef4-cb42-43ce-9b23-e090a4878d1c")); + + @MockBean + private CalculatedFieldLinkDao calculatedFieldLinkDao; + @SpyBean + private CalculatedFieldLinkDataValidator validator; + + @Test + public void testUpdateNonExistingCalculatedField() { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(CALCULATED_FIELD_LINK_ID); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(UUID.fromString("136477af-fd07-4498-b9c9-54fe50e82992"))); + + given(calculatedFieldLinkDao.findById(TENANT_ID, CALCULATED_FIELD_LINK_ID.getId())).willReturn(null); + + assertThatThrownBy(() -> validator.validateUpdate(TENANT_ID, calculatedFieldLink)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't update non existing calculated field link!"); + } + +} \ No newline at end of file From ac20551bf41d43a5e094f394a9f1d889c36e0b02 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 5 Nov 2024 17:05:10 +0200 Subject: [PATCH 007/416] added support of housekeeper deletion --- .../controller/CalculatedFieldController.java | 13 +++- ...CalculatedFieldsDeletionTaskProcessor.java | 43 +++++++++++ .../permission/CustomerUserPermissions.java | 31 +++++++- .../security/permission/Operation.java | 2 +- .../permission/TenantAdminPermissions.java | 21 +++-- .../CalculatedFieldControllerTest.java | 1 - .../CalculatedFieldService.java | 5 ++ .../data/housekeeper/HousekeeperTask.java | 4 + .../data/housekeeper/HousekeeperTaskType.java | 3 +- .../dao/asset/AssetProfileServiceImpl.java | 4 +- .../server/dao/asset/BaseAssetService.java | 4 +- .../BaseCalculatedFieldService.java | 31 +++++++- .../calculated_field/CalculatedFieldDao.java | 9 +++ .../dao/device/DeviceProfileServiceImpl.java | 4 +- .../server/dao/device/DeviceServiceImpl.java | 4 +- .../dao/entity/AbstractEntityService.java | 5 ++ .../dao/housekeeper/CleanUpService.java | 1 + .../CalculatedFieldRepository.java | 6 ++ .../JpaCalculatedFieldDao.java | 16 ++++ .../dao/service/AssetProfileServiceTest.java | 21 +++++ .../server/dao/service/AssetServiceTest.java | 77 ++++++++++++------- .../CalculatedFieldServiceTest.java | 5 +- .../dao/service/DeviceProfileServiceTest.java | 25 +++++- .../server/dao/service/DeviceServiceTest.java | 20 +++++ .../CalculatedFieldLinkDataValidatorTest.java | 2 +- 25 files changed, 300 insertions(+), 57 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java rename dao/src/test/java/org/thingsboard/server/dao/{calculated_field => service}/CalculatedFieldServiceTest.java (97%) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 26a3539a46..b3f2c018cb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -30,11 +30,11 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -63,7 +63,7 @@ public class CalculatedFieldController extends BaseController { public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); - checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); + checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); return calculatedFieldService.save(calculatedField); } @@ -76,7 +76,10 @@ public class CalculatedFieldController extends BaseController { public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); - return checkCalculatedFieldId(calculatedFieldId, Operation.READ); + CalculatedField calculatedField = calculatedFieldService.findById(getTenantId(), calculatedFieldId); + checkNotNull(calculatedField); + checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); + return calculatedField; } @@ -88,7 +91,9 @@ public class CalculatedFieldController extends BaseController { public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedField) throws Exception { checkParameter(CALCULATED_FIELD_ID, strCalculatedField); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedField)); - checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); + TenantId tenantId = getTenantId(); + CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java new file mode 100644 index 0000000000..18d84b02ce --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; + +@Component +@RequiredArgsConstructor +@Slf4j +public class CalculatedFieldsDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final CalculatedFieldService calculatedFieldService; + + @Override + public void process(HousekeeperTask task) throws Exception { + int deletedCount = calculatedFieldService.deleteAllCalculatedFieldsByEntityId(task.getTenantId(), task.getEntityId()); + log.debug("[{}][{}][{}] Deleted {} calculated fields", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_CALCULATED_FIELDS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 46cd1c2979..0c6dbc1599 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -34,8 +34,8 @@ public class CustomerUserPermissions extends AbstractPermissions { public CustomerUserPermissions() { super(); put(Resource.ALARM, customerAlarmPermissionChecker); - put(Resource.ASSET, customerEntityPermissionChecker); - put(Resource.DEVICE, customerEntityPermissionChecker); + put(Resource.ASSET, customerEntityWithCalculatedFieldPermissionChecker); + put(Resource.DEVICE, customerEntityWithCalculatedFieldPermissionChecker); put(Resource.CUSTOMER, customerPermissionChecker); put(Resource.DASHBOARD, customerDashboardPermissionChecker); put(Resource.ENTITY_VIEW, customerEntityPermissionChecker); @@ -85,6 +85,29 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; + private static final PermissionChecker customerEntityWithCalculatedFieldPermissionChecker = + new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_CREDENTIALS, + Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY, Operation.RPC_CALL, Operation.CLAIM_DEVICES, + Operation.WRITE, Operation.WRITE_ATTRIBUTES, Operation.WRITE_TELEMETRY, Operation.READ_CALCULATED_FIELD, + Operation.WRITE_CALCULATED_FIELD) { + + @Override + @SuppressWarnings("unchecked") + public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { + + if (!super.hasPermission(user, operation, entityId, entity)) { + return false; + } + if (!user.getTenantId().equals(entity.getTenantId())) { + return false; + } + if (!(entity instanceof HasCustomerId)) { + return false; + } + return operation.equals(Operation.CLAIM_DEVICES) || user.getCustomerId().equals(((HasCustomerId) entity).getCustomerId()); + } + }; + private static final PermissionChecker customerPermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY) { @@ -188,7 +211,8 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; - private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ) { + private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker( + Operation.READ, Operation.READ_CALCULATED_FIELD, Operation.WRITE_CALCULATED_FIELD) { @Override @SuppressWarnings("unchecked") @@ -202,4 +226,5 @@ public class CustomerUserPermissions extends AbstractPermissions { return user.getTenantId().equals(entity.getTenantId()); } }; + } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java index 9465a91cc8..d029d78cff 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java @@ -19,6 +19,6 @@ public enum Operation { ALL, CREATE, READ, WRITE, DELETE, ASSIGN_TO_CUSTOMER, UNASSIGN_FROM_CUSTOMER, RPC_CALL, READ_CREDENTIALS, WRITE_CREDENTIALS, READ_ATTRIBUTES, WRITE_ATTRIBUTES, READ_TELEMETRY, WRITE_TELEMETRY, CLAIM_DEVICES, - ASSIGN_TO_TENANT + ASSIGN_TO_TENANT, READ_CALCULATED_FIELD, WRITE_CALCULATED_FIELD } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 897581cc91..c730637148 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -23,15 +23,15 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -@Component(value="tenantAdminPermissions") +@Component(value = "tenantAdminPermissions") public class TenantAdminPermissions extends AbstractPermissions { public TenantAdminPermissions() { super(); put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker); put(Resource.ALARM, tenantEntityPermissionChecker); - put(Resource.ASSET, tenantEntityPermissionChecker); - put(Resource.DEVICE, tenantEntityPermissionChecker); + put(Resource.ASSET, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.DEVICE, tenantEntityWithCalculatedFieldPermissionChecker); put(Resource.CUSTOMER, tenantEntityPermissionChecker); put(Resource.DASHBOARD, tenantEntityPermissionChecker); put(Resource.ENTITY_VIEW, tenantEntityPermissionChecker); @@ -40,8 +40,8 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.USER, userPermissionChecker); put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker); - put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); - put(Resource.ASSET_PROFILE, tenantEntityPermissionChecker); + put(Resource.DEVICE_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.ASSET_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker); put(Resource.TB_RESOURCE, tbResourcePermissionChecker); put(Resource.OTA_PACKAGE, tenantEntityPermissionChecker); @@ -51,14 +51,23 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.VERSION_CONTROL, PermissionChecker.allowAllPermissionChecker); put(Resource.NOTIFICATION, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_SETTINGS, new PermissionChecker.GenericPermissionChecker(Operation.READ)); - put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { @Override public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { + if (!user.getTenantId().equals(entity.getTenantId())) { + return false; + } + return !Operation.READ_CALCULATED_FIELD.equals(operation) && !Operation.WRITE_CALCULATED_FIELD.equals(operation); + } + }; + + public static final PermissionChecker tenantEntityWithCalculatedFieldPermissionChecker = new PermissionChecker() { + @Override + public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { if (!user.getTenantId().equals(entity.getTenantId())) { return false; } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 685a58d48c..d9f4ac0377 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -115,7 +115,6 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) .andExpect(status().isOk()); doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); - } private CalculatedField getCalculatedField(DeviceId deviceId) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java index e1779e1d33..f271f4ed24 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; @@ -29,6 +30,10 @@ public interface CalculatedFieldService extends EntityDaoService { void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 8757b1121e..7a87528553 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -81,6 +81,10 @@ public class HousekeeperTask implements Serializable { return new TenantEntitiesDeletionHousekeeperTask(tenantId, entityType); } + public static HousekeeperTask deleteCalculatedFields(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_CALCULATED_FIELDS); + } + @JsonIgnore public String getDescription() { return taskType.getDescription() + " for " + entityId.getEntityType().getNormalName().toLowerCase() + " " + entityId.getId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java index edd13b3d5d..236844c17d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -30,7 +30,8 @@ public enum HousekeeperTaskType { DELETE_ALARMS("alarms deletion"), UNASSIGN_ALARMS("alarms unassigning"), DELETE_TENANT_ENTITIES("tenant entities deletion"), - DELETE_ENTITIES("entities deletion"); + DELETE_ENTITIES("entities deletion"), + DELETE_CALCULATED_FIELDS("calculated fields deletion"); private final String description; diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index 7d4d5300f9..baf20219ac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -192,8 +192,8 @@ public class AssetProfileServiceImpl extends CachedVersionedEntityService calculatedFieldDataValidator; private final DataValidator calculatedFieldLinkDataValidator; @@ -72,7 +79,7 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing findById, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + log.trace("Executing findById, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); @@ -80,12 +87,21 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing deleteRpc, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); } + @Override + public int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing deleteAllCalculatedFieldsByEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(entityId.getId(), id -> "Incorrect entityId " + id); + List calculatedFields = calculatedFieldDao.removeAllByEntityId(tenantId, entityId); + return calculatedFields.size(); + } + @Override public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); @@ -98,6 +114,11 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } } + @Override + public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); @@ -114,6 +135,12 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); + case ASSET_PROFILE -> + Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); + case DEVICE_PROFILE -> + Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' is not supported."); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java index e4bd019b18..d0d0a3c386 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -16,7 +16,16 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.List; + public interface CalculatedFieldDao extends Dao { + + boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); + + List removeAllByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 09e2be01e0..84dc1a7d14 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -230,8 +230,8 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService { + + boolean existsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java index 33cf926eaf..9a0eef26c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -15,16 +15,21 @@ */ package org.thingsboard.server.dao.sql.calculated_field; +import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.UUID; @Slf4j @@ -35,6 +40,17 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao removeAllByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldRepository.removeAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + @Override protected Class getEntityClass() { return CalculatedFieldEntity.class; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java index 7271a0fc8a..c11d8c6e1c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java @@ -28,10 +28,12 @@ import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; @@ -43,6 +45,7 @@ import java.util.concurrent.Executors; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class AssetProfileServiceTest extends AbstractServiceTest { @@ -54,6 +57,8 @@ public class AssetProfileServiceTest extends AbstractServiceTest { AssetProfileService assetProfileService; @Autowired AssetService assetService; + @Autowired + private CalculatedFieldService calculatedFieldService; @Test public void testSaveAssetProfile() { @@ -380,5 +385,21 @@ public class AssetProfileServiceTest extends AbstractServiceTest { assertThat(assetProfileInfos).isEqualTo(expected); } + @Test + public void testDeleteAssetProfileIfCalculatedFieldExists() { + AssetProfile assetProfile = this.createAssetProfile(tenantId, "Asset Profile"); + AssetProfile savedAssetProfile = assetProfileService.saveAssetProfile(assetProfile); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedAssetProfile.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> assetProfileService.deleteAssetProfile(tenantId, savedAssetProfile.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Deletion of Asset Profile is prohibited!"); + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 1976f173a5..e1e1bf70ed 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -39,6 +40,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; @@ -47,6 +49,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @DaoSqlTest @@ -63,6 +66,8 @@ public class AssetServiceTest extends AbstractServiceTest { @Autowired private AssetProfileService assetProfileService; @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired private PlatformTransactionManager platformTransactionManager; private IdComparator idComparator = new IdComparator<>(); @@ -214,24 +219,24 @@ public class AssetServiceTest extends AbstractServiceTest { public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); try { - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset B"+i); + asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(assetService.saveAsset(asset)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset C"+i); + asset.setName("My asset C" + i); asset.setType("typeC"); assets.add(assetService.saveAsset(asset)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset A"+i); + asset.setName("My asset A" + i); asset.setType("typeA"); assets.add(assetService.saveAsset(asset)); } @@ -267,10 +272,10 @@ public class AssetServiceTest extends AbstractServiceTest { @Test public void testFindAssetsByTenantId() { List assets = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); assets.add(assetService.saveAsset(asset)); } @@ -303,11 +308,11 @@ public class AssetServiceTest extends AbstractServiceTest { public void testFindAssetsByTenantIdAndName() { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -315,11 +320,11 @@ public class AssetServiceTest extends AbstractServiceTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -381,11 +386,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -394,11 +399,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -464,10 +469,10 @@ public class AssetServiceTest extends AbstractServiceTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); asset = assetService.saveAsset(asset); assets.add(new AssetInfo(assetService.assignAssetToCustomer(tenantId, asset.getId(), customerId), customer.getTitle(), customer.isPublic(), "default")); @@ -508,11 +513,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -521,11 +526,11 @@ public class AssetServiceTest extends AbstractServiceTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -596,11 +601,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -610,11 +615,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -848,4 +853,24 @@ public class AssetServiceTest extends AbstractServiceTest { ); } + @Test + public void testDeleteAssetIfCalculatedFieldExists() { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("My asset"); + asset.setType("default"); + Asset savedAsset = assetService.saveAsset(asset); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedAsset.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> assetService.deleteAsset(tenantId, savedAsset.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete asset that has entity views or calculated fields!"); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java similarity index 97% rename from dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java rename to dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index fdff4bc2f6..46cc478904 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.service; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -28,10 +28,9 @@ import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.service.AbstractServiceTest; -import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.UUID; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java index aa14e20151..9bb1abc3c8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java @@ -31,9 +31,11 @@ import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; @@ -49,6 +51,7 @@ import java.util.concurrent.Executors; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; @DaoSqlTest @@ -60,6 +63,9 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { DeviceService deviceService; @Autowired OtaPackageService otaPackageService; + @Autowired + private CalculatedFieldService calculatedFieldService; + private IdComparator idComparator = new IdComparator<>(); private IdComparator deviceProfileInfoIdComparator = new IdComparator<>(); @@ -397,7 +403,7 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { var profileA = deviceProfileService.saveDeviceProfile( - createDeviceProfile(tenantId, "profile A")); + createDeviceProfile(tenantId, "profile A")); deviceProfiles.add(deviceProfileService.saveDeviceProfile(profileA)); @@ -478,4 +484,21 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { assertThat(deviceProfileInfos).isEqualTo(expected); } + @Test + public void testDeleteDeviceProfileIfCalculatedFieldExists() { + DeviceProfile deviceProfile = this.createDeviceProfile(tenantId, "Device Profile"); + DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedDeviceProfile.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> deviceProfileService.deleteDeviceProfile(tenantId, savedDeviceProfile.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Deletion of Device Profile is prohibited!"); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 9fe3f85463..f2508090b2 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; @@ -87,6 +89,8 @@ public class DeviceServiceTest extends AbstractServiceTest { @Autowired TenantProfileService tenantProfileService; @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired private PlatformTransactionManager platformTransactionManager; @SpyBean private DeviceCredentialsDataValidator validator; @@ -1198,4 +1202,20 @@ public class DeviceServiceTest extends AbstractServiceTest { ); } + @Test + public void testDeleteDeviceIfCalculatedFieldExists() { + Device device = saveDevice(tenantId, "Test"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(device.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> deviceService.deleteDevice(tenantId, device.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete device that has entity views or calculated fields!"); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java index a74c59d7ad..c9fa57a751 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -54,4 +54,4 @@ public class CalculatedFieldLinkDataValidatorTest { .hasMessage("Can't update non existing calculated field link!"); } -} \ No newline at end of file +} From c223f1d2ba6754b739ba36f29f1468c4bea2d684 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 11:06:05 +0200 Subject: [PATCH 008/416] declined permissions for customers --- .../permission/CustomerUserPermissions.java | 27 ++----------------- .../permission/TenantAdminPermissions.java | 19 +++---------- .../dao/asset/AssetProfileServiceImpl.java | 4 +-- .../dao/device/DeviceProfileServiceImpl.java | 4 +-- .../dao/service/AssetProfileServiceTest.java | 22 --------------- .../dao/service/DeviceProfileServiceTest.java | 20 -------------- 6 files changed, 10 insertions(+), 86 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 0c6dbc1599..90019968df 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -34,8 +34,8 @@ public class CustomerUserPermissions extends AbstractPermissions { public CustomerUserPermissions() { super(); put(Resource.ALARM, customerAlarmPermissionChecker); - put(Resource.ASSET, customerEntityWithCalculatedFieldPermissionChecker); - put(Resource.DEVICE, customerEntityWithCalculatedFieldPermissionChecker); + put(Resource.ASSET, customerEntityPermissionChecker); + put(Resource.DEVICE, customerEntityPermissionChecker); put(Resource.CUSTOMER, customerPermissionChecker); put(Resource.DASHBOARD, customerDashboardPermissionChecker); put(Resource.ENTITY_VIEW, customerEntityPermissionChecker); @@ -85,29 +85,6 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; - private static final PermissionChecker customerEntityWithCalculatedFieldPermissionChecker = - new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_CREDENTIALS, - Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY, Operation.RPC_CALL, Operation.CLAIM_DEVICES, - Operation.WRITE, Operation.WRITE_ATTRIBUTES, Operation.WRITE_TELEMETRY, Operation.READ_CALCULATED_FIELD, - Operation.WRITE_CALCULATED_FIELD) { - - @Override - @SuppressWarnings("unchecked") - public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { - - if (!super.hasPermission(user, operation, entityId, entity)) { - return false; - } - if (!user.getTenantId().equals(entity.getTenantId())) { - return false; - } - if (!(entity instanceof HasCustomerId)) { - return false; - } - return operation.equals(Operation.CLAIM_DEVICES) || user.getCustomerId().equals(((HasCustomerId) entity).getCustomerId()); - } - }; - private static final PermissionChecker customerPermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index c730637148..de11521e85 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -30,8 +30,8 @@ public class TenantAdminPermissions extends AbstractPermissions { super(); put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker); put(Resource.ALARM, tenantEntityPermissionChecker); - put(Resource.ASSET, tenantEntityWithCalculatedFieldPermissionChecker); - put(Resource.DEVICE, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.ASSET, tenantEntityPermissionChecker); + put(Resource.DEVICE, tenantEntityPermissionChecker); put(Resource.CUSTOMER, tenantEntityPermissionChecker); put(Resource.DASHBOARD, tenantEntityPermissionChecker); put(Resource.ENTITY_VIEW, tenantEntityPermissionChecker); @@ -40,8 +40,8 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.USER, userPermissionChecker); put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker); - put(Resource.DEVICE_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); - put(Resource.ASSET_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); + put(Resource.ASSET_PROFILE, tenantEntityPermissionChecker); put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker); put(Resource.TB_RESOURCE, tbResourcePermissionChecker); put(Resource.OTA_PACKAGE, tenantEntityPermissionChecker); @@ -55,17 +55,6 @@ public class TenantAdminPermissions extends AbstractPermissions { public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { - @Override - public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { - if (!user.getTenantId().equals(entity.getTenantId())) { - return false; - } - return !Operation.READ_CALCULATED_FIELD.equals(operation) && !Operation.WRITE_CALCULATED_FIELD.equals(operation); - } - }; - - public static final PermissionChecker tenantEntityWithCalculatedFieldPermissionChecker = new PermissionChecker() { - @Override public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { if (!user.getTenantId().equals(entity.getTenantId())) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index baf20219ac..7d4d5300f9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -192,8 +192,8 @@ public class AssetProfileServiceImpl extends CachedVersionedEntityService assetProfileService.deleteAssetProfile(tenantId, savedAssetProfile.getId())) - .isInstanceOf(DataValidationException.class) - .hasMessage("Deletion of Asset Profile is prohibited!"); - } - } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java index 9bb1abc3c8..d99eaa2cea 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java @@ -63,9 +63,6 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { DeviceService deviceService; @Autowired OtaPackageService otaPackageService; - @Autowired - private CalculatedFieldService calculatedFieldService; - private IdComparator idComparator = new IdComparator<>(); private IdComparator deviceProfileInfoIdComparator = new IdComparator<>(); @@ -484,21 +481,4 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { assertThat(deviceProfileInfos).isEqualTo(expected); } - @Test - public void testDeleteDeviceProfileIfCalculatedFieldExists() { - DeviceProfile deviceProfile = this.createDeviceProfile(tenantId, "Device Profile"); - DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); - - CalculatedField calculatedField = new CalculatedField(); - calculatedField.setTenantId(tenantId); - calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); - calculatedField.setEntityId(savedDeviceProfile.getId()); - calculatedFieldService.save(calculatedField); - - assertThatThrownBy(() -> deviceProfileService.deleteDeviceProfile(tenantId, savedDeviceProfile.getId())) - .isInstanceOf(DataValidationException.class) - .hasMessage("Deletion of Device Profile is prohibited!"); - } - } From 3c073bad6e65d7f07e7fee4b1cbc9ecbacf3b9ad Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 11:11:34 +0200 Subject: [PATCH 009/416] removed authority customer user for endpoints --- .../server/controller/CalculatedFieldController.java | 4 ++-- .../server/dao/device/DeviceProfileServiceImpl.java | 4 ++-- .../server/dao/service/AssetProfileServiceTest.java | 1 + .../server/dao/service/DeviceProfileServiceTest.java | 5 +---- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index b3f2c018cb..78331e61b4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -57,7 +57,7 @@ public class CalculatedFieldController extends BaseController { "Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + "Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) @ResponseBody public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") @@ -70,7 +70,7 @@ public class CalculatedFieldController extends BaseController { @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." ) - @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) @ResponseBody public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index ec68562f5d..09e2be01e0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -230,8 +230,8 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService Date: Wed, 6 Nov 2024 15:21:28 +0200 Subject: [PATCH 010/416] added CF config and converter for it --- .../controller/CalculatedFieldController.java | 27 ++++ .../CalculatedFieldControllerTest.java | 38 ++++-- .../CalculatedFieldService.java | 2 + .../calculated_field/CalculatedField.java | 7 +- .../CalculatedFieldConfig.java | 43 +++++++ .../calculated_field/CalculatedFieldLink.java | 8 +- .../server/dao/asset/BaseAssetService.java | 4 +- .../BaseCalculatedFieldService.java | 35 ++++-- .../CalculatedFieldConfigUtil.java | 117 ++++++++++++++++++ .../calculated_field/CalculatedFieldDao.java | 2 + .../CalculatedFieldLinkDao.java | 4 +- .../dao/customer/CustomerServiceImpl.java | 4 + .../server/dao/device/DeviceServiceImpl.java | 4 +- .../dao/model/sql/CalculatedFieldEntity.java | 6 +- .../model/sql/CalculatedFieldLinkEntity.java | 6 +- .../CalculatedFieldLinkRepository.java | 2 +- .../CalculatedFieldRepository.java | 2 + .../JpaCalculatedFieldDao.java | 5 + .../JpaCalculatedFieldLinkDao.java | 6 +- .../server/dao/service/AssetServiceTest.java | 40 +++++- .../service/CalculatedFieldServiceTest.java | 52 +++++--- .../dao/service/CustomerServiceTest.java | 57 ++++++++- .../server/dao/service/DeviceServiceTest.java | 34 ++++- 23 files changed, 431 insertions(+), 74 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 78331e61b4..c33fb86ef8 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -27,14 +27,24 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; + +import java.util.EnumSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -46,6 +56,9 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI @Slf4j public class CalculatedFieldController extends BaseController { + private static final Set supportedEntityTypesForReferencedEntities = EnumSet.of( + EntityType.TENANT, EntityType.CUSTOMER, EntityType.ASSET, EntityType.DEVICE); + private final CalculatedFieldService calculatedFieldService; public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; @@ -64,6 +77,7 @@ public class CalculatedFieldController extends BaseController { @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); + checkReferencedEntities(calculatedField.getConfiguration()); return calculatedFieldService.save(calculatedField); } @@ -97,4 +111,17 @@ public class CalculatedFieldController extends BaseController { calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); } + private void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() + .map(CalculatedFieldConfig.Argument::getEntityId) + .filter(Objects::nonNull) + .toList(); + for (EntityId referencedEntityId : referencedEntityIds) { + if (!supportedEntityTypesForReferencedEntities.contains(referencedEntityId.getEntityType())) { + throw new IllegalArgumentException("Calculated fields do not support entity type '" + referencedEntityId.getEntityType() + "' for referenced entities."); + } + checkEntityId(referencedEntityId, Operation.READ); + } + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index d9f4ac0377..52271bef83 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -18,15 +18,18 @@ package org.thingsboard.server.controller; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.service.DaoSqlTest; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -75,7 +78,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); - assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(getCalculatedFieldConfig(testDevice.getId())); assertThat(savedCalculatedField.getVersion()).isEqualTo(calculatedField.getVersion()); savedCalculatedField.setName("Test CF"); @@ -123,19 +126,28 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + - " \"T\": {\n" + - " \"key\": \"temperature\",\n" + - " \"type\": \"TIME_SERIES\"\n" + - " },\n" + - " \"H\": {\n" + - " \"key\": \"humidity\",\n" + - " \"type\": \"TIME_SERIES\",\n" + - " \"defaultValue\": 50\n" + - " }\n" + - " }\n")); + calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } + private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(referencedEntityId); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + return config; + } + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java index f271f4ed24..b60c25e749 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -36,4 +36,6 @@ public interface CalculatedFieldService extends EntityDaoService { boolean existsByEntityId(TenantId tenantId, EntityId entityId); + boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java index 0bd8ded8f4..2482a7a19a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java @@ -47,8 +47,8 @@ public class CalculatedField extends BaseData implements HasN private String name; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; - @Schema(description = "JSON with the calculated field configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) - private transient JsonNode configuration; + @Schema + private transient CalculatedFieldConfig configuration; @Getter @Setter private Long version; @@ -64,8 +64,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, JsonNode configuration, Long version, CalculatedFieldId externalId) { - super(); + public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfig configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java new file mode 100644 index 0000000000..b1d63a9396 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.calculated_field; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Map; + +@Data +public class CalculatedFieldConfig { + + private Map arguments; + private Output output; + + @Data + public static class Argument { + private EntityId entityId; + private String key; + private String type; + private int defaultValue; + } + + @Data + public static class Output { + private String type; + private String expression; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java index 79f05c9b58..f24746abc0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java @@ -37,8 +37,8 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; - @Schema(description = "JSON with the calculated field link configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) - private transient JsonNode configuration; + @Schema + private transient CalculatedFieldConfig configuration; public CalculatedFieldLink() { super(); @@ -48,11 +48,11 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, JsonNode configuration, CalculatedFieldId calculatedFieldId) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldConfig configuration) { this.tenantId = tenantId; this.entityId = entityId; - this.configuration = configuration; this.calculatedFieldId = calculatedFieldId; + this.configuration = configuration; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index da86d82cf0..f4c68aa15b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -222,8 +222,8 @@ public class BaseAssetService extends AbstractCachedEntityService !referencedEntityId.equals(calculatedField.getEntityId())) + .map(CalculatedField::getConfiguration) + .map(CalculatedFieldConfig::getArguments) + .flatMap(arguments -> arguments.values().stream()) + .anyMatch(argument -> referencedEntityId.equals(argument.getEntityId())); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); @@ -142,22 +153,26 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); default -> - throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' is not supported."); + throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = calculatedFieldLinkDao.findCalculatedFieldLinkByEntityId(tenantId.getId(), calculatedField.getEntityId().getId()); - saveCalculatedFieldLink(tenantId, Objects.requireNonNullElseGet(calculatedFieldLink, () -> createCalculatedFieldLink(tenantId, calculatedField))); + CalculatedFieldLink existingLink = (calculatedField.getId() != null) + ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) + : null; + + CalculatedFieldLink updatedLink = buildCalculatedFieldLink(tenantId, calculatedField, existingLink); + saveCalculatedFieldLink(tenantId, updatedLink); } - private CalculatedFieldLink createCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); - calculatedFieldLink.setTenantId(tenantId); - calculatedFieldLink.setEntityId(calculatedField.getEntityId()); - calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); - calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); - return calculatedFieldLink; + private CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField, CalculatedFieldLink existingLink) { + CalculatedFieldLink link = (existingLink != null) ? existingLink : new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(calculatedField.getEntityId()); + link.setCalculatedFieldId(calculatedField.getId()); + link.setConfiguration(calculatedField.getConfiguration()); + return link; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java new file mode 100644 index 0000000000..8f9422b354 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +public class CalculatedFieldConfigUtil { + + public static CalculatedFieldConfig toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { + if (config == null) { + return null; + } + try { + CalculatedFieldConfig calculatedFieldConfig = new CalculatedFieldConfig(); + Map arguments = new HashMap<>(); + + JsonNode argumentsNode = config.get("arguments"); + if (argumentsNode != null && argumentsNode.isObject()) { + argumentsNode.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode argumentNode = entry.getValue(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + if (argumentNode.has("entityType") && argumentNode.has("entityId")) { + String referencedEntityType = argumentNode.get("entityType").asText(); + UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + } else { + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + } + argument.setKey(argumentNode.get("key").asText()); + argument.setType(argumentNode.get("type").asText()); + + if (argumentNode.has("defaultValue")) { + argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); + } + + arguments.put(key, argument); + }); + } + calculatedFieldConfig.setArguments(arguments); + + JsonNode outputNode = config.get("output"); + if (outputNode != null) { + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType(outputNode.get("type").asText()); + output.setExpression(outputNode.get("expression").asText()); + calculatedFieldConfig.setOutput(output); + } + + return calculatedFieldConfig; + + } catch (Exception e) { + throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); + } + } + + public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfig calculatedFieldConfig, EntityType entityType, UUID entityId) { + if (calculatedFieldConfig == null) { + return null; + } + try { + ObjectNode configNode = JacksonUtil.newObjectNode(); + + ObjectNode argumentsNode = configNode.putObject("arguments"); + calculatedFieldConfig.getArguments().forEach((key, argument) -> { + ObjectNode argumentNode = argumentsNode.putObject(key); + EntityId referencedEntityId = argument.getEntityId(); + if (referencedEntityId != null) { + argumentNode.put("entityType", referencedEntityId.getEntityType().name()); + argumentNode.put("entityId", referencedEntityId.getId().toString()); + } else { + argumentNode.put("entityType", entityType.name()); + argumentNode.put("entityId", entityId.toString()); + } + argumentNode.put("key", argument.getKey()); + argumentNode.put("type", argument.getType()); + argumentNode.put("defaultValue", argument.getDefaultValue()); + }); + + if (calculatedFieldConfig.getOutput() != null) { + ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("type", calculatedFieldConfig.getOutput().getType()); + outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); + } + + return configNode; + + } catch (Exception e) { + throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java index d0d0a3c386..e4ec365f46 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -26,6 +26,8 @@ public interface CalculatedFieldDao extends Dao { boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); + List findAllByTenantId(TenantId tenantId); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java index 1c422311eb..380a3b4d96 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java @@ -16,12 +16,14 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import java.util.UUID; public interface CalculatedFieldLinkDao extends Dao { - CalculatedFieldLink findCalculatedFieldLinkByEntityId(UUID tenantId, UUID entityId); + CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 2067ef4a5d..955d16c8e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -184,6 +184,10 @@ public class CustomerServiceImpl extends AbstractCachedEntityService implem this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedField.getConfiguration(); + this.configuration = calculatedFieldConfigToJson(calculatedField.getConfiguration(), entityType, entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -107,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(configuration); + calculatedField.setConfiguration(toCalculatedFieldConfig(configuration, entityType, entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 9f2efb230b..46c81e4449 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -34,6 +34,8 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; +import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; +import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; @@ -75,7 +77,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity { - CalculatedFieldLinkEntity findByTenantIdAndEntityId(UUID tenantId, UUID entityId); + CalculatedFieldLinkEntity findByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java index 17d7c0a9c6..d446d59859 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java @@ -25,6 +25,8 @@ public interface CalculatedFieldRepository extends JpaRepository findAllByTenantId(UUID tenantId); + List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java index 9a0eef26c3..8a873e1580 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -45,6 +45,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId) { + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantId(tenantId.getId())); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java index 0721e08d7f..4424f5bf42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java @@ -20,6 +20,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; @@ -37,8 +39,8 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao { assetService.deleteAsset(tenantId, asset.getId()); }); + assets.forEach((asset) -> { + assetService.deleteAsset(tenantId, asset.getId()); + }); } } @@ -854,23 +858,49 @@ public class AssetServiceTest extends AbstractServiceTest { } @Test - public void testDeleteAssetIfCalculatedFieldExists() { + public void testDeleteAssetIfReferencedInCalculatedField() { Asset asset = new Asset(); asset.setTenantId(tenantId); asset.setName("My asset"); asset.setType("default"); Asset savedAsset = assetService.saveAsset(asset); + Asset assetWithCf = new Asset(); + assetWithCf.setTenantId(tenantId); + assetWithCf.setName("Asset with CF"); + assetWithCf.setType("default"); + Asset savedAssetWithCf = assetService.saveAsset(assetWithCf); + CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); calculatedField.setType("Simple"); - calculatedField.setEntityId(savedAsset.getId()); - calculatedFieldService.save(calculatedField); + calculatedField.setEntityId(savedAssetWithCf.getId()); + + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(savedAsset.getId()); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThatThrownBy(() -> assetService.deleteAsset(tenantId, savedAsset.getId())) .isInstanceOf(DataValidationException.class) - .hasMessage("Can't delete asset that has entity views or calculated fields!"); + .hasMessage("Can't delete asset that has entity views or is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 46cc478904..32b7cbc00a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -21,17 +21,19 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; +import java.util.Map; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -60,7 +62,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedField() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThat(savedCalculatedField).isNotNull(); @@ -84,7 +86,8 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatesFieldWithNonExistingDeviceId() { - CalculatedField calculatedField = getCalculatedField(new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22"))); + DeviceId deviceId = new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22")); + CalculatedField calculatedField = getCalculatedField(deviceId, deviceId); assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) .isInstanceOf(IllegalArgumentException.class) @@ -94,7 +97,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedFieldWithExistingName() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); calculatedFieldService.save(calculatedField); assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) @@ -105,7 +108,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedFieldWithExistingExternalId() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); calculatedField.setExternalId(new CalculatedFieldId(UUID.fromString("2ef69d0a-89cf-4868-86f8-c50551d87ebe"))); calculatedFieldService.save(calculatedField); @@ -147,28 +150,18 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { private CalculatedField saveValidCalculatedField() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); return calculatedFieldService.save(calculatedField); } - private CalculatedField getCalculatedField(DeviceId deviceId) { + private CalculatedField getCalculatedField(EntityId entityId, EntityId referencedEntityId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); - calculatedField.setEntityId(deviceId); + calculatedField.setEntityId(entityId); calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + - " \"T\": {\n" + - " \"key\": \"temperature\",\n" + - " \"type\": \"TIME_SERIES\"\n" + - " },\n" + - " \"H\": {\n" + - " \"key\": \"humidity\",\n" + - " \"type\": \"TIME_SERIES\",\n" + - " \"defaultValue\": 50\n" + - " }\n" + - " }\n")); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } @@ -177,11 +170,30 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); calculatedFieldLink.setTenantId(tenantId); calculatedFieldLink.setEntityId(calculatedField.getEntityId()); - calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); +// calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); return calculatedFieldLink; } + private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(referencedEntityId); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + return config; + } + private Device createTestDevice() { Device device = new Device(); device.setTenantId(tenantId); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 0f4c139ce8..c021512a44 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -30,14 +30,20 @@ import org.testcontainers.shaded.org.awaitility.Awaitility; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -45,13 +51,17 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class CustomerServiceTest extends AbstractServiceTest { @Autowired CustomerService customerService; + @Autowired + CalculatedFieldService calculatedFieldService; + @Autowired + AssetService assetService; static final int TIMEOUT = 30; @@ -343,4 +353,49 @@ public class CustomerServiceTest extends AbstractServiceTest { } } + @Test + public void testDeleteCustomerIfReferencedInCalculatedField() { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle("My customer"); + Customer savedCustomer = customerService.saveCustomer(customer); + + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("My asset"); + asset.setType("default"); + Asset savedAsset = assetService.saveAsset(asset); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedAsset.getId()); + + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(savedCustomer.getId()); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> customerService.deleteCustomer(tenantId, savedCustomer.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete customer that is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index f2508090b2..a9ed4d201f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -66,6 +67,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -1203,19 +1205,41 @@ public class DeviceServiceTest extends AbstractServiceTest { } @Test - public void testDeleteDeviceIfCalculatedFieldExists() { - Device device = saveDevice(tenantId, "Test"); + public void testDeleteAssetIfReferencedInCalculatedField() { + Device device = saveDevice(tenantId, "Test Device"); + Device deviceWithCf = saveDevice(tenantId, "Device with CF"); CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); calculatedField.setType("Simple"); - calculatedField.setEntityId(device.getId()); - calculatedFieldService.save(calculatedField); + calculatedField.setEntityId(deviceWithCf.getId()); + + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(device.getId()); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThatThrownBy(() -> deviceService.deleteDevice(tenantId, device.getId())) .isInstanceOf(DataValidationException.class) - .hasMessage("Can't delete device that has entity views or calculated fields!"); + .hasMessage("Can't delete device that has entity views or is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } + } From 47e7c0f3a535aaa16d2d4b9bd74fe9f1f1f556f4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 15:25:07 +0200 Subject: [PATCH 011/416] removed permission for customer --- .../service/security/permission/CustomerUserPermissions.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 90019968df..46cd1c2979 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -188,8 +188,7 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; - private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker( - Operation.READ, Operation.READ_CALCULATED_FIELD, Operation.WRITE_CALCULATED_FIELD) { + private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ) { @Override @SuppressWarnings("unchecked") @@ -203,5 +202,4 @@ public class CustomerUserPermissions extends AbstractPermissions { return user.getTenantId().equals(entity.getTenantId()); } }; - } From 258ba4807ba0c514091a8bc2f8a475cf6699b701 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 16:29:20 +0200 Subject: [PATCH 012/416] fixed entity service registry test --- .../server/dao/entity/DefaultEntityServiceRegistry.java | 3 +++ .../server/dao/service/EntityServiceRegistryTest.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java index 364d3a12f3..56e8b95473 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java @@ -43,6 +43,9 @@ public class DefaultEntityServiceRegistry implements EntityServiceRegistry { if (EntityType.RULE_CHAIN.equals(entityType)) { entityDaoServicesMap.put(EntityType.RULE_NODE, entityDaoService); } + if (EntityType.CALCULATED_FIELD.equals(entityType)) { + entityDaoServicesMap.put(EntityType.CALCULATED_FIELD_LINK, entityDaoService); + } }); log.debug("Initialized EntityServiceRegistry total [{}] entries", entityDaoServicesMap.size()); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java index c3f6213398..0769479f48 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java @@ -20,6 +20,7 @@ import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.dao.rule.RuleChainService; @@ -44,4 +45,8 @@ public class EntityServiceRegistryTest extends AbstractServiceTest { Assert.assertTrue(entityServiceRegistry.getServiceByEntityType(EntityType.RULE_NODE) instanceof RuleChainService); } + @Test + public void givenCalculatedFieldLinkEntityType_whenGetServiceByEntityTypeCalled_thenReturnedCalculatedFieldService() { + Assert.assertTrue(entityServiceRegistry.getServiceByEntityType(EntityType.CALCULATED_FIELD_LINK) instanceof CalculatedFieldService); + } } From 1d702365381db46e7a925043973ba2d5fe021134 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 7 Nov 2024 09:14:41 +0200 Subject: [PATCH 013/416] fixed entity dao registry test --- .../dao/calculated_field/CalculatedFieldLinkDao.java | 2 -- .../dao/sql/calculated_field/JpaCalculatedFieldDao.java | 7 +++++++ .../sql/calculated_field/JpaCalculatedFieldLinkDao.java | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java index 380a3b4d96..7fa0285b9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java @@ -20,8 +20,6 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; -import java.util.UUID; - public interface CalculatedFieldLinkDao extends Dao { CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java index 8a873e1580..affd652514 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -20,6 +20,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -65,4 +66,10 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao getRepository() { return calculatedFieldRepository; } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java index 4424f5bf42..f77baa4e42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; @@ -53,4 +54,9 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao Date: Thu, 7 Nov 2024 12:02:39 +0200 Subject: [PATCH 014/416] renamed directories --- .../thingsboard/server/controller/BaseController.java | 4 ++-- .../server/controller/CalculatedFieldController.java | 8 +++----- .../processor/CalculatedFieldsDeletionTaskProcessor.java | 2 +- .../server/controller/CalculatedFieldControllerTest.java | 4 ++-- .../{calculated_field => cf}/CalculatedFieldService.java | 6 +++--- .../data/{calculated_field => cf}/CalculatedField.java | 3 +-- .../{calculated_field => cf}/CalculatedFieldConfig.java | 2 +- .../{calculated_field => cf}/CalculatedFieldLink.java | 3 +-- .../BaseCalculatedFieldService.java | 9 ++++----- .../CalculatedFieldConfigUtil.java | 4 ++-- .../dao/{calculated_field => cf}/CalculatedFieldDao.java | 4 ++-- .../{calculated_field => cf}/CalculatedFieldLinkDao.java | 4 ++-- .../server/dao/entity/AbstractEntityService.java | 2 +- .../server/dao/model/sql/CalculatedFieldEntity.java | 6 +++--- .../server/dao/model/sql/CalculatedFieldLinkEntity.java | 6 +++--- .../service/validator/CalculatedFieldDataValidator.java | 4 ++-- .../validator/CalculatedFieldLinkDataValidator.java | 4 ++-- .../CalculatedFieldLinkRepository.java | 2 +- .../CalculatedFieldRepository.java | 2 +- .../{calculated_field => cf}/JpaCalculatedFieldDao.java | 6 +++--- .../JpaCalculatedFieldLinkDao.java | 6 +++--- .../thingsboard/server/dao/service/AssetServiceTest.java | 6 +++--- .../server/dao/service/CalculatedFieldServiceTest.java | 8 ++++---- .../server/dao/service/CustomerServiceTest.java | 6 +++--- .../server/dao/service/DeviceServiceTest.java | 6 +++--- .../server/dao/service/EntityServiceRegistryTest.java | 2 +- .../validator/CalculatedFieldDataValidatorTest.java | 4 ++-- .../validator/CalculatedFieldLinkDataValidatorTest.java | 4 ++-- 28 files changed, 61 insertions(+), 66 deletions(-) rename common/dao-api/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldService.java (87%) rename common/data/src/main/java/org/thingsboard/server/common/data/{calculated_field => cf}/CalculatedField.java (97%) rename common/data/src/main/java/org/thingsboard/server/common/data/{calculated_field => cf}/CalculatedFieldConfig.java (94%) rename common/data/src/main/java/org/thingsboard/server/common/data/{calculated_field => cf}/CalculatedFieldLink.java (95%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/BaseCalculatedFieldService.java (96%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldConfigUtil.java (97%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldDao.java (89%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldLinkDao.java (88%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/CalculatedFieldLinkRepository.java (94%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/CalculatedFieldRepository.java (95%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/JpaCalculatedFieldDao.java (92%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/JpaCalculatedFieldLinkDao.java (91%) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 6d52bdb358..eb6fec88b5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -67,7 +67,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; @@ -126,7 +126,7 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index c33fb86ef8..2e9989334d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -28,23 +28,21 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import java.util.EnumSet; import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java index 18d84b02ce..2362081ff9 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java @@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; @Component @RequiredArgsConstructor diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 52271bef83..affab6ee8b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,8 +21,8 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java similarity index 87% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index b60c25e749..8171759def 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java similarity index 97% rename from common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 2482a7a19a..5c52ad0609 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.calculated_field; +package org.thingsboard.server.common.data.cf; -import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java index b1d63a9396..b51258b2ec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.calculated_field; +package org.thingsboard.server.common.data.cf; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java similarity index 95% rename from common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index f24746abc0..2f176b13d2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.calculated_field; +package org.thingsboard.server.common.data.cf; -import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java similarity index 96% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0b2ba43fbb..fe4422ee71 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -37,7 +37,6 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; -import java.util.Objects; import java.util.Optional; import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java similarity index 97% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java index 8f9422b354..34dd885e12 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java similarity index 89% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index e4ec365f46..b755d4b6c0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java similarity index 88% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 7fa0285b9a..95259e75c4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index cda9222f76..7cf7b445d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.alarm.AlarmService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.DataValidationException; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 43b4f2fc9f..98eb050d28 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -23,7 +23,7 @@ import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -33,8 +33,8 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.toCalculatedFieldConfig; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 46c81e4449..73a2f9fbdf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -23,7 +23,7 @@ import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -34,8 +34,8 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.toCalculatedFieldConfig; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index e50f2c902a..80e421f350 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -17,9 +17,9 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java index 83c30d4d03..2b11f8fe42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java @@ -17,9 +17,9 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java similarity index 94% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index a4f035c2c9..fa1c5ce38e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java similarity index 95% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index d446d59859..333057e8c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java similarity index 92% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index affd652514..3fbfe92efe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; @@ -21,11 +21,11 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java similarity index 91% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index f77baa4e42..b9ed6dc3a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 39386e4668..ba0c73fb09 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,8 +30,8 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -41,7 +41,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 32b7cbc00a..632e4b19e6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,13 +23,13 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index c021512a44..548a5bfcf5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,13 +31,13 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index a9ed4d201f..16bb08350b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,8 +39,8 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -52,7 +52,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java index 0769479f48..147e69d1cb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java @@ -20,7 +20,7 @@ import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.dao.rule.RuleChainService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java index bc7990ca64..6d0eb0ad38 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -19,10 +19,10 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.UUID; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java index c9fa57a751..5ec6841150 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -19,11 +19,11 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.UUID; From 4dbc35273ae9182024829ca558336b92fce5de6d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 7 Nov 2024 13:26:48 +0200 Subject: [PATCH 015/416] added calculated field service to ctx --- .../server/actors/ActorSystemContext.java | 5 +++++ .../actors/ruleChain/DefaultTbContext.java | 6 ++++++ .../server/dao/cf/CalculatedFieldService.java | 3 +++ .../dao/cf/BaseCalculatedFieldService.java | 9 +++++++++ .../thingsboard/rule/engine/api/TbContext.java | 3 +++ .../rule/engine/util/TenantIdLoader.java | 14 ++++++++++++++ .../rule/engine/util/TenantIdLoaderTest.java | 17 +++++++++++++++++ 7 files changed, 57 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 5966716ed8..400374c731 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -62,6 +62,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; @@ -389,6 +390,10 @@ public class ActorSystemContext { @Getter private SlackService slackService; + @Autowired + @Getter + private CalculatedFieldService calculatedFieldService; + @Lazy @Autowired(required = false) @Getter diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index ec10402821..274d50a6c9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -78,6 +78,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -848,6 +849,11 @@ class DefaultTbContext implements TbContext { return mainCtx.getSlackService(); } + @Override + public CalculatedFieldService getCalculatedFieldService() { + return mainCtx.getCalculatedFieldService(); + } + @Override public boolean isExternalNodeForceAck() { return mainCtx.isExternalNodeForceAck(); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 8171759def..b2da1d3fe4 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; @@ -34,6 +35,8 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index fe4422ee71..be21c9b887 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -114,6 +115,14 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } } + @Override + public CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { + log.trace("Executing findCalculatedFieldLinkById, tenantId [{}], calculatedFieldLinkId [{}]", tenantId, calculatedFieldLinkId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); + return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); + } + @Override public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 7dd4505f29..22c7ced237 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -50,6 +50,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -357,6 +358,8 @@ public interface TbContext { SlackService getSlackService(); + CalculatedFieldService getCalculatedFieldService(); + boolean isExternalNodeForceAck(); /** diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java index 1e1b031259..98724b7b68 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java @@ -18,10 +18,13 @@ package org.thingsboard.rule.engine.util; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -157,6 +160,17 @@ public class TenantIdLoader { case MOBILE_APP: tenantEntity = ctx.getMobileAppService().findMobileAppById(ctxTenantId, new MobileAppId(id)); break; + case CALCULATED_FIELD: + tenantEntity = ctx.getCalculatedFieldService().findById(ctxTenantId, new CalculatedFieldId(id)); + break; + case CALCULATED_FIELD_LINK: + CalculatedFieldLink calculatedFieldLink = ctx.getCalculatedFieldService().findCalculatedFieldLinkById(ctxTenantId, new CalculatedFieldLinkId(id)); + if (calculatedFieldLink != null) { + tenantEntity = ctx.getCalculatedFieldService().findById(ctxTenantId, calculatedFieldLink.getCalculatedFieldId()); + } else { + tenantEntity = null; + } + break; default: throw new RuntimeException("Unexpected entity type: " + entityId.getEntityType()); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index 3e0e479c80..2234e50a16 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -43,6 +43,8 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -66,6 +68,7 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; @@ -151,6 +154,8 @@ public class TenantIdLoaderTest { private DomainService domainService; @Mock private MobileAppService mobileAppService; + @Mock + private CalculatedFieldService calculatedFieldService; private TenantId tenantId; private TenantProfileId tenantProfileId; @@ -392,6 +397,18 @@ public class TenantIdLoaderTest { when(ctx.getMobileAppService()).thenReturn(mobileAppService); doReturn(mobileApp).when(mobileAppService).findMobileAppById(eq(tenantId), any()); break; + case CALCULATED_FIELD: + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); + doReturn(calculatedField).when(calculatedFieldService).findById(eq(tenantId), any()); + break; + case CALCULATED_FIELD_LINK: + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); + doReturn(calculatedFieldLink).when(calculatedFieldService).findCalculatedFieldLinkById(eq(tenantId), any()); + break; default: throw new RuntimeException("Unexpected originator EntityType " + entityType); } From bcef71f78d6ceb6a36703aa11660ea9a0383df06 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 7 Nov 2024 14:07:58 +0200 Subject: [PATCH 016/416] added tbCalculatedField Service --- .../controller/CalculatedFieldController.java | 39 +---- .../entitiy/AbstractTbEntityService.java | 47 ++++++ .../cf/DefaultTbCalculatedFieldService.java | 149 ++++++++++++++++++ .../entitiy/cf/TbCalculatedFieldService.java | 33 ++++ .../server/dao/cf/CalculatedFieldService.java | 6 + .../dao/cf/BaseCalculatedFieldService.java | 42 ++--- .../server/dao/cf/CalculatedFieldDao.java | 2 + .../server/dao/cf/CalculatedFieldLinkDao.java | 4 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 + .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 6 + .../service/CalculatedFieldServiceTest.java | 10 -- 11 files changed, 270 insertions(+), 73 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 2e9989334d..4bf243034b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -27,23 +27,14 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; -import java.util.EnumSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; - import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -54,10 +45,7 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI @Slf4j public class CalculatedFieldController extends BaseController { - private static final Set supportedEntityTypesForReferencedEntities = EnumSet.of( - EntityType.TENANT, EntityType.CUSTOMER, EntityType.ASSET, EntityType.DEVICE); - - private final CalculatedFieldService calculatedFieldService; + private final TbCalculatedFieldService tbCalculatedFieldService; public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; @@ -75,8 +63,7 @@ public class CalculatedFieldController extends BaseController { @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); - checkReferencedEntities(calculatedField.getConfiguration()); - return calculatedFieldService.save(calculatedField); + return tbCalculatedFieldService.save(calculatedField, getCurrentUser()); } @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", @@ -88,7 +75,7 @@ public class CalculatedFieldController extends BaseController { public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); - CalculatedField calculatedField = calculatedFieldService.findById(getTenantId(), calculatedFieldId); + CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); checkNotNull(calculatedField); checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); return calculatedField; @@ -103,23 +90,9 @@ public class CalculatedFieldController extends BaseController { public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedField) throws Exception { checkParameter(CALCULATED_FIELD_ID, strCalculatedField); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedField)); - TenantId tenantId = getTenantId(); - CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); - calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); - } - - private void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig) throws ThingsboardException { - List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() - .map(CalculatedFieldConfig.Argument::getEntityId) - .filter(Objects::nonNull) - .toList(); - for (EntityId referencedEntityId : referencedEntityIds) { - if (!supportedEntityTypesForReferencedEntities.contains(referencedEntityId.getEntityType())) { - throw new IllegalArgumentException("Calculated fields do not support entity type '" + referencedEntityId.getEntityType() + "' for referenced entities."); - } - checkEntityId(referencedEntityId, Operation.READ); - } + tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 438cb58b91..2b19d34e57 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -25,16 +25,30 @@ import org.springframework.context.annotation.Lazy; import org.springframework.core.env.Environment; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.util.ThrowingBiFunction; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.AccessControlService; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -43,6 +57,8 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import static org.thingsboard.server.dao.service.Validator.validateId; + @Slf4j public abstract class AbstractTbEntityService { @@ -71,6 +87,18 @@ public abstract class AbstractTbEntityService { @Autowired(required = false) @Lazy private EntitiesVersionControlService vcService; + @Autowired + protected AccessControlService accessControlService; + @Autowired + protected TenantService tenantService; + @Autowired + protected AssetService assetService; + @Autowired + protected DeviceService deviceService; + @Autowired + protected AssetProfileService assetProfileService; + @Autowired + protected DeviceProfileService deviceProfileService; protected boolean isTestProfile() { return Set.of(this.env.getActiveProfiles()).contains("test"); @@ -120,4 +148,23 @@ public abstract class AbstractTbEntityService { return Futures.immediateFailedFuture(new RuntimeException("Operation not supported!")); } } + + protected & HasTenantId, I extends EntityId> E checkEntityId(I entityId, ThrowingBiFunction findingFunction, Operation operation, SecurityUser user) throws Exception { + try { + validateId((UUIDBased) entityId, "Invalid entity id"); + E entity = findingFunction.apply(user.getTenantId(), entityId); + checkNotNull(entity, entityId.getEntityType().getNormalName() + " with id [" + entityId + "] is not found"); + return checkEntity(user, entity, operation); + } catch (Exception e) { + throw e; + } + } + + + protected & HasTenantId, I extends EntityId> E checkEntity(SecurityUser user, E entity, Operation operation) throws ThingsboardException { + checkNotNull(entity, "Entity not found"); + accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation, entity.getId(), entity); + return entity; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java new file mode 100644 index 0000000000..a7c77cf4bc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -0,0 +1,149 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.thingsboard.server.dao.service.Validator.validateEntityId; + +@TbCoreComponent +@Service +@Slf4j +public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { + + private final Map calculatedFields; + private final Map calculatedFieldLinks; + private final CalculatedFieldService calculatedFieldService; + + public DefaultTbCalculatedFieldService(CalculatedFieldService calculatedFieldService) { + this.calculatedFields = calculatedFieldService.findAll().stream().collect(Collectors.toMap(CalculatedField::getId, cf -> cf)); + this.calculatedFieldLinks = calculatedFieldService.findAllCalculatedFieldLinks().stream().collect(Collectors.toMap(CalculatedFieldLink::getId, cfl -> cfl)); + this.calculatedFieldService = calculatedFieldService; + } + + @Override + public void onMsg() { + + } + + @Override + public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { + ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; + TenantId tenantId = calculatedField.getTenantId(); + try { + checkEntityExistence(tenantId, calculatedField.getEntityId()); + checkReferencedEntities(calculatedField.getConfiguration(), user); + CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); + logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); + return savedCalculatedField; + } catch (ThingsboardException e) { + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), calculatedField, actionType, user, e); + throw e; + } + } + + @Override + public CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user) { + return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); + } + + @Override + @Transactional + public void delete(CalculatedField calculatedField, SecurityUser user) { + ActionType actionType = ActionType.DELETED; + TenantId tenantId = calculatedField.getTenantId(); + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + try { + calculatedFieldService.deleteCalculatedField(tenantId, calculatedFieldId); + logEntityActionService.logEntityAction(tenantId, calculatedFieldId, calculatedField, actionType, user, calculatedFieldId.toString()); + } catch (Exception e) { + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), actionType, user, e, calculatedFieldId.toString()); + throw e; + } + } + + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { + switch (entityId.getEntityType()) { + case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); + case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); + case ASSET_PROFILE -> + Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); + case DEVICE_PROFILE -> + Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); + default -> + throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); + } + } + + private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig, SecurityUser user) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() + .map(CalculatedFieldConfig.Argument::getEntityId) + .filter(Objects::nonNull) + .toList(); + for (EntityId referencedEntityId : referencedEntityIds) { + validateEntityId(referencedEntityId, id -> "Invalid entity id " + id); + E entity = findEntity(user.getTenantId(), referencedEntityId); + checkNotNull(entity); + checkEntity(user, entity, Operation.READ); + } + + } + + private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { + return (E) switch (entityId.getEntityType()) { + case TENANT -> tenantService.findTenantById((TenantId) entityId); + case CUSTOMER -> customerService.findCustomerById(tenantId, (CustomerId) entityId); + case ASSET -> assetService.findAssetById(tenantId, (AssetId) entityId); + case DEVICE -> deviceService.findDeviceById(tenantId, (DeviceId) entityId); + default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java new file mode 100644 index 0000000000..2dc1cc35b2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.service.security.model.SecurityUser; + +public interface TbCalculatedFieldService { + + void onMsg(); + + CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; + + CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); + + void delete(CalculatedField calculatedField, SecurityUser user); + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index b2da1d3fe4..7760c49d2d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -23,12 +23,16 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; +import java.util.List; + public interface CalculatedFieldService extends EntityDaoService { CalculatedField save(CalculatedField calculatedField); CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAll(); + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); @@ -37,6 +41,8 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + List findAllCalculatedFieldLinks(); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index be21c9b887..ecc5aecce1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -22,19 +22,11 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.asset.AssetProfileService; -import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.device.DeviceProfileService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; @@ -53,10 +45,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { private final CalculatedFieldDao calculatedFieldDao; private final CalculatedFieldLinkDao calculatedFieldLinkDao; - private final DeviceService deviceService; - private final AssetService assetService; - private final DeviceProfileService deviceProfileService; - private final AssetProfileService assetProfileService; private final DataValidator calculatedFieldDataValidator; private final DataValidator calculatedFieldLinkDataValidator; @@ -65,7 +53,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); try { TenantId tenantId = calculatedField.getTenantId(); - checkEntityExistence(tenantId, calculatedField.getEntityId()); log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); @@ -86,6 +73,12 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); } + @Override + public List findAll() { + log.trace("Executing findAll"); + return calculatedFieldDao.findAll(); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); @@ -123,6 +116,12 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); } + @Override + public List findAllCalculatedFieldLinks() { + log.trace("Executing findAllCalculatedFieldLinks"); + return calculatedFieldLinkDao.findAll(); + } + @Override public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); @@ -148,23 +147,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return EntityType.CALCULATED_FIELD; } - private void checkEntityExistence(TenantId tenantId, EntityId entityId) { - switch (entityId.getEntityType()) { - case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); - case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); - case ASSET_PROFILE -> - Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); - case DEVICE_PROFILE -> - Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); - default -> - throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); - } - } - private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { CalculatedFieldLink existingLink = (calculatedField.getId() != null) ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index b755d4b6c0..bb52d0e2c2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -28,6 +28,8 @@ public interface CalculatedFieldDao extends Dao { List findAllByTenantId(TenantId tenantId); + List findAll(); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 95259e75c4..c4a06c88a3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -20,8 +20,12 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.List; + public interface CalculatedFieldLinkDao extends Dao { CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAll(); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 3fbfe92efe..4bf4c1ead7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -51,6 +51,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAll() { + return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index b9ed6dc3a4..f584a8d76c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.UUID; @Slf4j @@ -44,6 +45,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findAll() { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); + } + @Override protected Class getEntityClass() { return CalculatedFieldLinkEntity.class; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 632e4b19e6..c4ad2856a9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -84,16 +84,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } - @Test - public void testSaveCalculatesFieldWithNonExistingDeviceId() { - DeviceId deviceId = new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22")); - CalculatedField calculatedField = getCalculatedField(deviceId, deviceId); - - assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Device with id [" + calculatedField.getEntityId().getId() + "] does not exist."); - } - @Test public void testSaveCalculatedFieldWithExistingName() { Device device = createTestDevice(); From 86dd8e725bf701f0c0fe561c3e2b6e8729235305 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 11 Nov 2024 17:29:24 +0200 Subject: [PATCH 017/416] added onCalculatedFieldAdded implementation --- .../entitiy/AbstractTbEntityService.java | 3 + .../entitiy/cf/CalculatedFieldCtx.java | 31 ++ .../entitiy/cf/CalculatedFieldState.java | 30 ++ .../cf/DefaultTbCalculatedFieldService.java | 281 ++++++++++++++++-- .../entitiy/cf/TbCalculatedFieldService.java | 8 +- .../CalculatedFieldControllerTest.java | 2 +- .../server/dao/asset/AssetService.java | 2 + .../server/dao/cf/CalculatedFieldService.java | 10 +- .../server/dao/device/DeviceService.java | 2 + .../server/dao/entity/EntityService.java | 3 + .../cf/BaseCalculatedFieldConfiguration.java | 158 ++++++++++ .../common/data/cf/CalculatedField.java | 8 +- .../data/cf/CalculatedFieldConfiguration.java | 48 +++ .../common/data/cf/CalculatedFieldLink.java | 4 +- .../cf/CalculatedFiledLinkConfiguration.java | 29 ++ .../SimpleCalculatedFieldConfiguration.java | 39 +++ common/proto/src/main/proto/queue.proto | 21 ++ .../server/dao/asset/AssetDao.java | 10 + .../server/dao/asset/BaseAssetService.java | 9 + .../dao/cf/BaseCalculatedFieldService.java | 60 ++-- .../dao/cf/CalculatedFieldConfigUtil.java | 172 +++++------ .../server/dao/cf/CalculatedFieldDao.java | 4 + .../server/dao/cf/CalculatedFieldLinkDao.java | 6 +- .../server/dao/device/DeviceDao.java | 12 + .../server/dao/device/DeviceServiceImpl.java | 9 + .../server/dao/entity/BaseEntityService.java | 5 + .../dao/model/sql/CalculatedFieldEntity.java | 18 +- .../model/sql/CalculatedFieldLinkEntity.java | 9 +- .../server/dao/sql/asset/AssetRepository.java | 10 + .../server/dao/sql/asset/JpaAssetDao.java | 11 + .../sql/cf/CalculatedFieldLinkRepository.java | 3 +- ...efaultNativeCalculatedFieldRepository.java | 146 +++++++++ .../dao/sql/cf/JpaCalculatedFieldDao.java | 11 + .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 14 +- .../cf/NativeCalculatedFieldRepository.java | 29 ++ .../dao/sql/device/DeviceRepository.java | 8 + .../server/dao/sql/device/JpaDeviceDao.java | 11 + .../server/dao/service/AssetServiceTest.java | 2 +- .../service/CalculatedFieldServiceTest.java | 2 +- .../dao/service/CustomerServiceTest.java | 2 +- .../server/dao/service/DeviceServiceTest.java | 2 +- 41 files changed, 1085 insertions(+), 159 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 2b19d34e57..d2b3890c70 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -99,6 +100,8 @@ public abstract class AbstractTbEntityService { protected AssetProfileService assetProfileService; @Autowired protected DeviceProfileService deviceProfileService; + @Autowired + protected EntityService entityService; protected boolean isTestProfile() { return Set.of(this.env.getActiveProfiles()).contains("test"); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java new file mode 100644 index 0000000000..61f28304f2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@Builder +public class CalculatedFieldCtx { + + private final CalculatedFieldId calculatedFieldId; + private final EntityId entityId; + private final CalculatedFieldState state; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java new file mode 100644 index 0000000000..dc07b820ea --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Builder; +import lombok.Data; + +import java.util.Map; + +@Data +@Builder +public class CalculatedFieldState { + + Map arguments; + String result; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index a7c77cf4bc..836f17650e 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,36 +15,64 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -52,21 +80,213 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @TbCoreComponent @Service @Slf4j +@RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { - private final Map calculatedFields; - private final Map calculatedFieldLinks; private final CalculatedFieldService calculatedFieldService; + private final TbDeviceProfileCache deviceProfileCache; + private final TbAssetProfileCache assetProfileCache; + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private ListeningScheduledExecutorService scheduledExecutor; - public DefaultTbCalculatedFieldService(CalculatedFieldService calculatedFieldService) { - this.calculatedFields = calculatedFieldService.findAll().stream().collect(Collectors.toMap(CalculatedField::getId, cf -> cf)); - this.calculatedFieldLinks = calculatedFieldService.findAllCalculatedFieldLinks().stream().collect(Collectors.toMap(CalculatedFieldLink::getId, cfl -> cfl)); - this.calculatedFieldService = calculatedFieldService; + private ListeningExecutorService calculatedFieldExecutor; + private ListeningExecutorService calculatedFieldCallbackExecutor; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); + + @Value("${state.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @Value("10") + @Getter + private int defaultCalculatedFieldCheckIntervalInSec; + + @PostConstruct + public void init() { + // from AbstractPartitionBasedService + scheduledExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("calculated-field-scheduled"))); + /// + calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, new Random().nextInt(defaultCalculatedFieldCheckIntervalInSec), defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); + } + + @PreDestroy + public void stop() { + // from AbstractPartitionBasedService + if (scheduledExecutor != null) { + scheduledExecutor.shutdown(); + } + /// + if (calculatedFieldExecutor != null) { + calculatedFieldExecutor.shutdownNow(); + } + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + + private ListenableFuture> fetchAttributesForEntity(TenantId tenantId, EntityId entityId, List keys) { + return attributesService.find(tenantId, entityId, AttributeScope.SERVER_SCOPE, keys); + } + + private ListenableFuture> fetchTimeSeries(TenantId tenantId, EntityId entityId, List keys) { + return timeseriesService.findLatest(tenantId, entityId, keys); + } + + private ListenableFuture initializeStateFromFutures(TenantId tenantId, EntityId entityId, CalculatedField calculatedField, List attributeKeys, List timeSeriesKeys) { + ListenableFuture> attributesFuture = fetchAttributesForEntity(tenantId, entityId, attributeKeys); + ListenableFuture> timeSeriesFuture = fetchTimeSeries(tenantId, entityId, timeSeriesKeys); + + ListenableFuture> combinedFuture = Futures.allAsList(attributesFuture, timeSeriesFuture); + + return Futures.transform(combinedFuture, results -> { + List attributes = (List) results.get(0); + List timeSeries = (List) results.get(1); + + initializeState(calculatedField, attributes, timeSeries); + + return null; + }, MoreExecutors.directExecutor()); + } + + private void initializeState(CalculatedField calculatedField, List attributes, List timeSeries) { + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state != null) { + String calculation = performCalculation(state.getArguments()); + + Map updatedArguments = state.getArguments(); + + state = CalculatedFieldState.builder() + .arguments(updatedArguments) + .result(calculation) + .build(); + } else { + // initial calculation + Map arguments = calculatedField.getConfiguration().getArguments(); + + Map argumentValues = arguments.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> resolveArgumentValue(entry.getKey(), entry.getValue(), attributes, timeSeries) + )); + + String calculation = performCalculation(argumentValues); + + state = CalculatedFieldState.builder() + .arguments(argumentValues) + .result(calculation) + .build(); + } + + calculatedFieldCtx = new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), state); + states.put(calculatedField.getId(), calculatedFieldCtx); + } + + private String resolveArgumentValue(String key, BaseCalculatedFieldConfiguration.Argument argument, + List attributes, List timeSeries) { + String type = argument.getType(); + String value = null; + + if ("ATTRIBUTES".equals(type)) { + value = attributes.stream() + .filter(attribute -> attribute.getKey().equals(key)) + .map(AttributeKvEntry::getValueAsString) + .findFirst() + .orElse(null); + } else if ("TIME_SERIES".equals(type)) { + value = timeSeries.stream() + .filter(tsKvEntry -> tsKvEntry.getKey().equals(key)) + .map(TsKvEntry::getValueAsString) + .findFirst() + .orElse(null); + } + + return value != null ? value : argument.getDefaultValue(); + } + + @Override + public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cf != null) { + EntityId entityId = cf.getEntityId(); + calculatedFields.put(calculatedFieldId, cf); + calculatedFieldLinks.put(calculatedFieldId, links); + switch (entityId.getEntityType()) { + case ASSET, DEVICE: { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + case ASSET_PROFILE: { + PageDataIterable assetIds = new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); + for (AssetId assetId : assetIds) { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, assetId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + case DEVICE_PROFILE: { + PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); + for (DeviceId deviceId : deviceIds) { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, deviceId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + default: throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + } + } else { + //Calculated field or entity was probably deleted while message was in queue; + callback.onSuccess(); + } + } catch (Exception e) { + log.trace("Failed to process calculated field add msg: [{}]", proto, e); + callback.onFailure(e); + } } @Override - public void onMsg() { + public void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + } catch (Exception e) { + log.trace("Failed to process calculated field update msg: [{}]", proto, e); + callback.onFailure(e); + } + } + @Override + public void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback) { + try { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.remove(calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to process calculated field delete msg: [{}]", proto, e); + callback.onFailure(e); + } } @Override @@ -105,28 +325,26 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + // TODO: read all states(CalculatedFieldCtx) + states.keySet().removeIf(calculatedFieldId -> !calculatedFields.containsKey(calculatedFieldId)); + } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { - case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); - case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); - case ASSET_PROFILE -> - Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); - case DEVICE_PROFILE -> - Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } } - private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig, SecurityUser user) throws ThingsboardException { - List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() - .map(CalculatedFieldConfig.Argument::getEntityId) - .filter(Objects::nonNull) - .toList(); + private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getReferencedEntities(); for (EntityId referencedEntityId : referencedEntityIds) { validateEntityId(referencedEntityId, id -> "Invalid entity id " + id); E entity = findEntity(user.getTenantId(), referencedEntityId); @@ -137,13 +355,14 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { - return (E) switch (entityId.getEntityType()) { - case TENANT -> tenantService.findTenantById((TenantId) entityId); - case CUSTOMER -> customerService.findCustomerById(tenantId, (CustomerId) entityId); - case ASSET -> assetService.findAssetById(tenantId, (AssetId) entityId); - case DEVICE -> deviceService.findDeviceById(tenantId, (DeviceId) entityId); + return switch (entityId.getEntityType()) { + case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); }; } + private String performCalculation(Map argumentValues) { + return "calculation"; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 2dc1cc35b2..4f14270c3d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,11 +18,17 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onMsg(); + void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback); + + void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback); + + void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback); CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index affab6ee8b..3f505c78ef 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -126,7 +126,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(getCalculatedFieldConfig(null)); +// calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 7252db8097..2a9fe08827 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -63,6 +63,8 @@ public interface AssetService extends EntityDaoService { PageData findAssetInfosByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds); void deleteAssetsByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 7760c49d2d..c12acade6c 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.entity.EntityDaoService; import java.util.List; @@ -31,7 +33,9 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - List findAll(); + List findAllCalculatedFields(); + + PageData findAllCalculatedFields(PageLink pageLink); void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); @@ -43,6 +47,10 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinks(); + List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + PageData findAllCalculatedFieldLinks(PageLink pageLink); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index 005c740571..0848e5a1cb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -75,6 +75,8 @@ public interface DeviceService extends EntityDaoService { PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); + PageData findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type, PageLink pageLink); long countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType otaPackageType); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index 5f522121d7..fc8859eceb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.entity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -34,6 +35,8 @@ public interface EntityService { Optional fetchEntityCustomerId(TenantId tenantId, EntityId entityId); + Optional> fetchEntity(TenantId tenantId, EntityId entityId); + Optional fetchNameLabelAndCustomerDetails(TenantId tenantId, EntityId entityId); long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..9bb6d59428 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -0,0 +1,158 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@Data +public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + @JsonIgnore + private final ObjectMapper mapper = new ObjectMapper(); + + protected Map arguments; + protected SimpleCalculatedFieldConfiguration.Output output; + + public BaseCalculatedFieldConfiguration() { + } + + public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); + this.arguments = calculatedFieldConfig.getArguments(); + this.output = calculatedFieldConfig.getOutput(); + } + + @Override + public List getReferencedEntities() { + return arguments.values().stream() + .map(SimpleCalculatedFieldConfiguration.Argument::getEntityId) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId) { + CalculatedFiledLinkConfiguration linkConfiguration = new CalculatedFiledLinkConfiguration(); + arguments.values().stream() + .filter(argument -> argument.getEntityId().equals(entityId)) + .forEach(argument -> { + switch (argument.getType()) { + case "ATTRIBUTES": + linkConfiguration.getAttributes().add(argument.getKey()); + break; + case "TIME_SERIES": + linkConfiguration.getTimeSeries().add(argument.getKey()); + break; + } + }); + + return linkConfiguration; + } + + @Override + public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { + ObjectNode configNode = mapper.createObjectNode(); + + ObjectNode argumentsNode = configNode.putObject("arguments"); + arguments.forEach((key, argument) -> { + ObjectNode argumentNode = argumentsNode.putObject(key); + EntityId referencedEntityId = argument.getEntityId(); + if (referencedEntityId != null) { + argumentNode.put("entityType", referencedEntityId.getEntityType().name()); + argumentNode.put("entityId", referencedEntityId.getId().toString()); + } else { + argumentNode.put("entityType", entityType.name()); + argumentNode.put("entityId", entityId.toString()); + } + argumentNode.put("key", argument.getKey()); + argumentNode.put("type", argument.getType()); + argumentNode.put("defaultValue", argument.getDefaultValue()); + }); + + if (output != null) { + ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("type", output.getType()); + outputNode.put("expression", output.getExpression()); + } + + return configNode; + } + + @Data + public static class Argument { + private EntityId entityId; + private String key; + private String type; + private String defaultValue; + } + + @Data + public static class Output { + private String type; + private String expression; + } + + private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { + if (config == null || !config.isObject()) { + return null; + } + + Map arguments = new HashMap<>(); + JsonNode argumentsNode = config.get("arguments"); + if (argumentsNode != null && argumentsNode.isObject()) { + argumentsNode.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode argumentNode = entry.getValue(); + Argument argument = new Argument(); + if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { + String referencedEntityType = argumentNode.get("entityType").asText(); + UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + } else { + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + } + argument.setKey(argumentNode.get("key").asText()); + argument.setType(argumentNode.get("type").asText()); + argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + arguments.put(key, argument); + }); + } + this.setArguments(arguments); + + JsonNode outputNode = config.get("output"); + if (outputNode != null) { + Output output = new Output(); + output.setType(outputNode.get("type").asText()); + output.setExpression(outputNode.get("expression").asText()); + this.setOutput(output); + } + + return this; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 5c52ad0609..d96de37a39 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.io.Serializable; + @Schema @Data @EqualsAndHashCode(callSuper = true) @@ -46,8 +48,8 @@ public class CalculatedField extends BaseData implements HasN private String name; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; - @Schema - private transient CalculatedFieldConfig configuration; + @Schema(implementation = SimpleCalculatedFieldConfiguration.class) + private transient CalculatedFieldConfiguration configuration; @Getter @Setter private Long version; @@ -63,7 +65,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfig configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java new file mode 100644 index 0000000000..e599b8c4d3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE") +}) +public interface CalculatedFieldConfiguration { + + String getType(); + + Map getArguments(); + + List getReferencedEntities(); + + CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId); + + JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index 2f176b13d2..922fda1f34 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -37,7 +37,7 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; @Schema - private transient CalculatedFieldConfig configuration; + private transient CalculatedFiledLinkConfiguration configuration; public CalculatedFieldLink() { super(); @@ -47,7 +47,7 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldConfig configuration) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFiledLinkConfiguration configuration) { this.tenantId = tenantId; this.entityId = entityId; this.calculatedFieldId = calculatedFieldId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java new file mode 100644 index 0000000000..26d867fd7f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class CalculatedFiledLinkConfiguration { + + private List attributes = new ArrayList<>(); + private List timeSeries = new ArrayList<>(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..d635e0d82e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Data +public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public SimpleCalculatedFieldConfiguration() { + super(); + } + + public SimpleCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public String getType() { + return "SIMPLE"; + } +} diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 875a531566..e1d6320779 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1267,6 +1267,27 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } +message CalculatedFieldAddMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + +message CalculatedFieldUpdateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + +message CalculatedFieldDeleteMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + /** TB Core to Version Control Service */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 42ab2e2545..8e40ec6d87 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -103,6 +103,16 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD */ PageData findAssetInfosByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** + * Find asset ids by tenantId, assetProfileId and page link. + * + * @param tenantId the tenantId + * @param assetProfileId the assetProfileId + * @param pageLink the page link + * @return the list of asset objects + */ + PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** * Find assets by tenantId and assets Ids. * diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index f4c68aa15b..2135792174 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -285,6 +285,15 @@ public class BaseAssetService extends AbstractCachedEntityService findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink) { + log.trace("Executing findAssetIdsByTenantIdAndAssetProfileId, tenantId [{}], assetProfileId [{}]", tenantId, assetProfileId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(assetProfileId, id -> INCORRECT_ASSET_PROFILE_ID + id); + validatePageLink(pageLink); + return assetDao.findAssetIdsByTenantIdAndAssetProfileId(tenantId.getId(), assetProfileId.getId(), pageLink); + } + @Override public ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds) { log.trace("Executing findAssetsByTenantIdAndIdsAsync, tenantId [{}], assetIds [{}]", tenantId, assetIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index ecc5aecce1..0210f6d70b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -20,20 +20,24 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePageLink; @Service("CalculatedFieldDaoService") @Slf4j @@ -74,11 +78,18 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } @Override - public List findAll() { + public List findAllCalculatedFields() { log.trace("Executing findAll"); return calculatedFieldDao.findAll(); } + @Override + public PageData findAllCalculatedFields(PageLink pageLink) { + log.trace("Executing findAll, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldDao.findAll(pageLink); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); @@ -122,6 +133,19 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldLinkDao.findAll(); } + @Override + public List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksById, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); + } + + @Override + public PageData findAllCalculatedFieldLinks(PageLink pageLink) { + log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldLinkDao.findAll(pageLink); + } + @Override public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); @@ -132,9 +156,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldDao.findAllByTenantId(tenantId).stream() .filter(calculatedField -> !referencedEntityId.equals(calculatedField.getEntityId())) .map(CalculatedField::getConfiguration) - .map(CalculatedFieldConfig::getArguments) - .flatMap(arguments -> arguments.values().stream()) - .anyMatch(argument -> referencedEntityId.equals(argument.getEntityId())); + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } @Override @@ -148,21 +171,22 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink existingLink = (calculatedField.getId() != null) - ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) - : null; - - CalculatedFieldLink updatedLink = buildCalculatedFieldLink(tenantId, calculatedField, existingLink); - saveCalculatedFieldLink(tenantId, updatedLink); + List links = buildCalculatedFieldLinks(tenantId, calculatedField); + links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); } - private CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField, CalculatedFieldLink existingLink) { - CalculatedFieldLink link = (existingLink != null) ? existingLink : new CalculatedFieldLink(); - link.setTenantId(tenantId); - link.setEntityId(calculatedField.getEntityId()); - link.setCalculatedFieldId(calculatedField.getId()); - link.setConfiguration(calculatedField.getConfiguration()); - return link; + private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); + return cfConfig.getReferencedEntities().stream() + .map(referencedEntityId -> { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedField.getId()); + link.setConfiguration(cfConfig.getReferencedEntityConfig(referencedEntityId)); + return link; + }) + .collect(Collectors.toList()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java index 34dd885e12..980fff6273 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java @@ -19,7 +19,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -29,89 +31,89 @@ import java.util.UUID; public class CalculatedFieldConfigUtil { - public static CalculatedFieldConfig toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { - if (config == null) { - return null; - } - try { - CalculatedFieldConfig calculatedFieldConfig = new CalculatedFieldConfig(); - Map arguments = new HashMap<>(); - - JsonNode argumentsNode = config.get("arguments"); - if (argumentsNode != null && argumentsNode.isObject()) { - argumentsNode.fields().forEachRemaining(entry -> { - String key = entry.getKey(); - JsonNode argumentNode = entry.getValue(); - - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); - if (argumentNode.has("entityType") && argumentNode.has("entityId")) { - String referencedEntityType = argumentNode.get("entityType").asText(); - UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); - } else { - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - } - argument.setKey(argumentNode.get("key").asText()); - argument.setType(argumentNode.get("type").asText()); - - if (argumentNode.has("defaultValue")) { - argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); - } - - arguments.put(key, argument); - }); - } - calculatedFieldConfig.setArguments(arguments); - - JsonNode outputNode = config.get("output"); - if (outputNode != null) { - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); - output.setType(outputNode.get("type").asText()); - output.setExpression(outputNode.get("expression").asText()); - calculatedFieldConfig.setOutput(output); - } - - return calculatedFieldConfig; - - } catch (Exception e) { - throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); - } - } - - public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfig calculatedFieldConfig, EntityType entityType, UUID entityId) { - if (calculatedFieldConfig == null) { - return null; - } - try { - ObjectNode configNode = JacksonUtil.newObjectNode(); - - ObjectNode argumentsNode = configNode.putObject("arguments"); - calculatedFieldConfig.getArguments().forEach((key, argument) -> { - ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getEntityId(); - if (referencedEntityId != null) { - argumentNode.put("entityType", referencedEntityId.getEntityType().name()); - argumentNode.put("entityId", referencedEntityId.getId().toString()); - } else { - argumentNode.put("entityType", entityType.name()); - argumentNode.put("entityId", entityId.toString()); - } - argumentNode.put("key", argument.getKey()); - argumentNode.put("type", argument.getType()); - argumentNode.put("defaultValue", argument.getDefaultValue()); - }); - - if (calculatedFieldConfig.getOutput() != null) { - ObjectNode outputNode = configNode.putObject("output"); - outputNode.put("type", calculatedFieldConfig.getOutput().getType()); - outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); - } - - return configNode; - - } catch (Exception e) { - throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); - } - } +// public static CalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { +// if (config == null) { +// return null; +// } +// try { +// CalculatedFieldConfiguration calculatedFieldConfig = new BaseCalculatedFieldConfiguration(); +// Map arguments = new HashMap<>(); +// +// JsonNode argumentsNode = config.get("arguments"); +// if (argumentsNode != null && argumentsNode.isObject()) { +// argumentsNode.fields().forEachRemaining(entry -> { +// String key = entry.getKey(); +// JsonNode argumentNode = entry.getValue(); +// +// CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); +// if (argumentNode.has("entityType") && argumentNode.has("entityId")) { +// String referencedEntityType = argumentNode.get("entityType").asText(); +// UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); +// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); +// } else { +// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); +// } +// argument.setKey(argumentNode.get("key").asText()); +// argument.setType(argumentNode.get("type").asText()); +// +// if (argumentNode.has("defaultValue")) { +// argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); +// } +// +// arguments.put(key, argument); +// }); +// } +// calculatedFieldConfig.setArguments(arguments); +// +// JsonNode outputNode = config.get("output"); +// if (outputNode != null) { +// CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); +// output.setType(outputNode.get("type").asText()); +// output.setExpression(outputNode.get("expression").asText()); +// calculatedFieldConfig.setOutput(output); +// } +// +// return calculatedFieldConfig; +// +// } catch (Exception e) { +// throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); +// } +// } +// +// public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfiguration calculatedFieldConfig, EntityType entityType, UUID entityId) { +// if (calculatedFieldConfig == null) { +// return null; +// } +// try { +// ObjectNode configNode = JacksonUtil.newObjectNode(); +// +// ObjectNode argumentsNode = configNode.putObject("arguments"); +// calculatedFieldConfig.getArguments().forEach((key, argument) -> { +// ObjectNode argumentNode = argumentsNode.putObject(key); +// EntityId referencedEntityId = argument.getEntityId(); +// if (referencedEntityId != null) { +// argumentNode.put("entityType", referencedEntityId.getEntityType().name()); +// argumentNode.put("entityId", referencedEntityId.getId().toString()); +// } else { +// argumentNode.put("entityType", entityType.name()); +// argumentNode.put("entityId", entityId.toString()); +// } +// argumentNode.put("key", argument.getKey()); +// argumentNode.put("type", argument.getType()); +// argumentNode.put("defaultValue", argument.getDefaultValue()); +// }); +// +// if (calculatedFieldConfig.getOutput() != null) { +// ObjectNode outputNode = configNode.putObject("output"); +// outputNode.put("type", calculatedFieldConfig.getOutput().getType()); +// outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); +// } +// +// return configNode; +// +// } catch (Exception e) { +// throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); +// } +// } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index bb52d0e2c2..4abe02a09b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -18,6 +18,8 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; @@ -30,6 +32,8 @@ public interface CalculatedFieldDao extends Dao { List findAll(); + PageData findAll(PageLink pageLink); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index c4a06c88a3..728e19b890 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -18,14 +18,18 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; public interface CalculatedFieldLinkDao extends Dao { - CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); List findAll(); + PageData findAll(PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 2305e4419c..7c82df7703 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -22,7 +22,9 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; @@ -85,6 +87,16 @@ public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntit */ PageData findDevicesByTenantIdAndType(UUID tenantId, String type, PageLink pageLink); + /** + * Find device ids by tenantId, type and page link. + * + * @param tenantId the tenantId + * @param deviceProfileId the deviceProfileId + * @param pageLink the page link + * @return the list of device objects + */ + PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, OtaPackageType type, diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index ac6b8330a9..ffebd5f032 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -395,6 +395,15 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink) { + log.trace("Executing findDeviceIdsByTenantIdAndType, tenantId [{}], deviceProfileId [{}], pageLink [{}]", tenantId, deviceProfileId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(deviceProfileId, id -> INCORRECT_DEVICE_PROFILE_ID + id); + validatePageLink(pageLink); + return deviceDao.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId.getId(), deviceProfileId.getId(), pageLink); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3cdcf5e874..5efea9a6df 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -134,6 +134,11 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return fetchAndConvert(tenantId, entityId, this::getNameLabelAndCustomerDetails); } + @Override + public Optional> fetchEntity(TenantId tenantId, EntityId entityId) { + return fetchAndConvert(tenantId, entityId, Function.identity()); + } + private Optional fetchAndConvert(TenantId tenantId, EntityId entityId, Function, T> converter) { EntityDaoService entityDaoService = entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()); Optional> entityOpt = entityDaoService.findEntity(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 98eb050d28..9cb4f08640 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,6 +24,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -33,8 +35,6 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; @@ -93,7 +93,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedFieldConfigToJson(calculatedField.getConfiguration(), entityType, entityId); + this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(entityType, entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -109,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(toCalculatedFieldConfig(configuration, entityType, entityId)); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); @@ -117,4 +117,14 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + String type = config.get("type").asText(); + switch (type) { + case "SIMPLE": + return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + default: + throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 73a2f9fbdf..3dc08d6bf1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -22,8 +22,10 @@ import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -34,8 +36,6 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; @@ -65,7 +65,6 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity, Expor @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT a.id FROM AssetEntity a " + + "WHERE a.tenantId = :tenantId " + + "AND a.assetProfileId = :assetProfileId " + + "AND (:textSearch IS NULL OR ilike(a.type, CONCAT('%', :textSearch, '%')) = true) ") + Page findAssetIdsByTenantIdAndAssetProfileId(@Param("tenantId") UUID tenantId, + @Param("assetProfileId") UUID assetProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT a FROM AssetEntity a WHERE a.tenantId = :tenantId " + "AND a.customerId = :customerId AND a.type = :type " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 0ef4370f30..db6472f825 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -41,6 +41,7 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityInfosToDto; @@ -159,6 +160,16 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, AssetInfoEntity.assetInfoColumnMap))); } + @Override + public PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData(assetRepository.findAssetIdsByTenantIdAndAssetProfileId( + tenantId, + assetProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(AssetId::new); + } + @Override public PageData findAssetsByTenantIdAndCustomerIdAndType(UUID tenantId, UUID customerId, String type, PageLink pageLink) { return DaoUtil.toPageData(assetRepository diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index fa1c5ce38e..61c4026cca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -18,10 +18,11 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; +import java.util.List; import java.util.UUID; public interface CalculatedFieldLinkRepository extends JpaRepository { - CalculatedFieldLinkEntity findByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + List findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..8401d99128 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedFieldRepository { + + private final String CF_COUNT_QUERY = "SELECT count(id) FROM calculated_field;"; + private final String CF_QUERY = "SELECT * FROM calculated_field ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final String CFL_COUNT_QUERY = "SELECT count(id) FROM calculated_field_link;"; + private final String CFL_QUERY = "SELECT * FROM calculated_field_link ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + @Override + public PageData findCalculatedFields(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CF_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CF_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + String type = (String) row.get("type"); + String name = (String) row.get("name"); + int configurationVersion = (int) row.get("configuration_version"); + JsonNode configuration = JacksonUtil.valueToTree(row.get("configuration")); + long version = (long) row.get("version"); + Object externalIdObj = row.get("external_id"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setId(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(new TenantId(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(type); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); + calculatedField.setVersion(version); + calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); + + return calculatedField; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + @Override + public PageData findCalculatedFieldLinks(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CFL_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CFL_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + UUID calculatedFieldId = (UUID) row.get("calculated_field_id"); + JsonNode configuration = JacksonUtil.valueToTree(row.get("configuration")); + + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setId(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(new TenantId(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + calculatedFieldLink.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFiledLinkConfiguration.class)); + + return calculatedFieldLink; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + String type = config.get("type").asText(); + switch (type) { + case "SIMPLE": + return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + default: + throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 4bf4c1ead7..1137b91947 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -18,16 +18,20 @@ package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; @@ -40,6 +44,7 @@ import java.util.UUID; public class JpaCalculatedFieldDao extends JpaAbstractDao implements CalculatedFieldDao { private final CalculatedFieldRepository calculatedFieldRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; @Override public boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId) { @@ -56,6 +61,12 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAll(PageLink pageLink) { + log.debug("Try to find calculated fields by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFields(DaoUtil.toPageable(pageLink)); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index f584a8d76c..a2f8f224c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -22,7 +22,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; @@ -39,10 +42,11 @@ import java.util.UUID; public class JpaCalculatedFieldLinkDao extends JpaAbstractDao implements CalculatedFieldLinkDao { private final CalculatedFieldLinkRepository calculatedFieldLinkRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; @Override - public CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - return DaoUtil.getData(calculatedFieldLinkRepository.findByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); + public List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); } @Override @@ -50,6 +54,12 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findAll(PageLink pageLink) { + log.debug("Try to find calculated field links by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFieldLinks(DaoUtil.toPageable(pageLink)); + } + @Override protected Class getEntityClass() { return CalculatedFieldLinkEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..76fed8c311 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeCalculatedFieldRepository { + + PageData findCalculatedFields(Pageable pageable); + + PageData findCalculatedFieldLinks(Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index c55210b606..5093da78c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -81,6 +81,14 @@ public interface DeviceRepository extends JpaRepository, Exp @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT d.id FROM DeviceEntity d WHERE d.tenantId = :tenantId " + + "AND d.deviceProfileId = :deviceProfileId " + + "AND (:textSearch IS NULL OR ilike(d.type, CONCAT('%', :textSearch, '%')) = true)") + Page findIdsByTenantIdAndDeviceProfileId(@Param("tenantId") UUID tenantId, + @Param("deviceProfileId") UUID deviceProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT d FROM DeviceEntity d WHERE d.tenantId = :tenantId " + "AND d.deviceProfileId = :deviceProfileId " + "AND d.firmwareId IS NULL") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 48bb998016..bf0def9fca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -173,6 +173,17 @@ public class JpaDeviceDao extends JpaAbstractDao implement DaoUtil.toPageable(pageLink))); } + @Override + public PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData( + deviceRepository.findIdsByTenantIdAndDeviceProfileId( + tenantId, + deviceProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(DeviceId::new); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index ba0c73fb09..311bf23263 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -892,7 +892,7 @@ public class AssetServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index c4ad2856a9..8e50e1e0fb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -151,7 +151,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); +// calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 548a5bfcf5..3f990b9dd7 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -387,7 +387,7 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 16bb08350b..90d04d5216 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1230,7 +1230,7 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); From 444c5bf07976df4e8ac7d57fb870a120dd673551 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 12:25:59 +0200 Subject: [PATCH 018/416] refactored service and test classes --- .../cf/DefaultTbCalculatedFieldService.java | 48 ++++--- .../cf/BaseCalculatedFieldConfiguration.java | 12 +- .../common/data/cf/CalculatedFieldConfig.java | 43 ------- .../data/cf/CalculatedFieldConfiguration.java | 7 +- .../common/data/cf/CalculatedFieldLink.java | 4 +- ... => CalculatedFieldLinkConfiguration.java} | 2 +- .../dao/cf/BaseCalculatedFieldService.java | 9 +- .../dao/cf/CalculatedFieldConfigUtil.java | 119 ------------------ .../dao/model/sql/CalculatedFieldEntity.java | 3 +- .../model/sql/CalculatedFieldLinkEntity.java | 4 +- ...efaultNativeCalculatedFieldRepository.java | 9 +- .../main/resources/sql/schema-entities.sql | 4 +- .../server/dao/service/AssetServiceTest.java | 12 +- .../service/CalculatedFieldServiceTest.java | 36 +++--- .../dao/service/CustomerServiceTest.java | 12 +- .../server/dao/service/DeviceServiceTest.java | 13 +- 16 files changed, 84 insertions(+), 253 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{CalculatedFiledLinkConfiguration.java => CalculatedFieldLinkConfiguration.java} (94%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 836f17650e..e222259a53 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -38,7 +38,7 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -58,8 +58,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -84,8 +82,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { private final CalculatedFieldService calculatedFieldService; - private final TbDeviceProfileCache deviceProfileCache; - private final TbAssetProfileCache assetProfileCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private ListeningScheduledExecutorService scheduledExecutor; @@ -215,6 +211,15 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp return value != null ? value : argument.getDefaultValue(); } + private void initializeForProfile(TenantId tenantId, List links, CalculatedField cf, Iterable profileIds) { + for (T profileId : profileIds) { + for (CalculatedFieldLink link : links) { + CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, profileId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + @Override public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { try { @@ -227,33 +232,24 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE: { + case ASSET, DEVICE -> { for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); } } - case ASSET_PROFILE: { + case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - for (AssetId assetId : assetIds) { - for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, assetId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + initializeForProfile(tenantId, links, cf, assetIds); } - case DEVICE_PROFILE: { + case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - for (DeviceId deviceId : deviceIds) { - for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, deviceId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + initializeForProfile(tenantId, links, cf, deviceIds); } - default: throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + default -> + throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } } else { //Calculated field or entity was probably deleted while message was in queue; @@ -336,8 +332,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { - case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) - .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> + Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } @@ -357,7 +354,8 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); - default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); + default -> + throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); }; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 9bb6d59428..3894835abc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; @@ -37,7 +38,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel private final ObjectMapper mapper = new ObjectMapper(); protected Map arguments; - protected SimpleCalculatedFieldConfiguration.Output output; + protected Output output; public BaseCalculatedFieldConfiguration() { } @@ -45,19 +46,20 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); this.arguments = calculatedFieldConfig.getArguments(); - this.output = calculatedFieldConfig.getOutput(); + this.output = calculatedFieldConfig.getOutput(); } @Override public List getReferencedEntities() { return arguments.values().stream() - .map(SimpleCalculatedFieldConfiguration.Argument::getEntityId) + .map(Argument::getEntityId) + .filter(Objects::nonNull) .collect(Collectors.toList()); } @Override - public CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId) { - CalculatedFiledLinkConfiguration linkConfiguration = new CalculatedFiledLinkConfiguration(); + public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { + CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.values().stream() .filter(argument -> argument.getEntityId().equals(entityId)) .forEach(argument -> { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java deleted file mode 100644 index b51258b2ec..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.Map; - -@Data -public class CalculatedFieldConfig { - - private Map arguments; - private Output output; - - @Data - public static class Argument { - private EntityId entityId; - private String key; - private String type; - private int defaultValue; - } - - @Data - public static class Output { - private String type; - private String expression; - } - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index e599b8c4d3..deaeebcb50 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; @@ -35,14 +36,18 @@ import java.util.UUID; }) public interface CalculatedFieldConfiguration { + @JsonIgnore String getType(); Map getArguments(); + @JsonIgnore List getReferencedEntities(); - CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId); + @JsonIgnore + CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId); + @JsonIgnore JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index 922fda1f34..f9f9ee625f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -37,7 +37,7 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; @Schema - private transient CalculatedFiledLinkConfiguration configuration; + private transient CalculatedFieldLinkConfiguration configuration; public CalculatedFieldLink() { super(); @@ -47,7 +47,7 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFiledLinkConfiguration configuration) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldLinkConfiguration configuration) { this.tenantId = tenantId; this.entityId = entityId; this.calculatedFieldId = calculatedFieldId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index 26d867fd7f..02d668a67a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -21,7 +21,7 @@ import java.util.ArrayList; import java.util.List; @Data -public class CalculatedFiledLinkConfiguration { +public class CalculatedFieldLinkConfiguration { private List attributes = new ArrayList<>(); private List timeSeries = new ArrayList<>(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0210f6d70b..db5539eb96 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -110,13 +110,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); - try { - log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); - return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); - } catch (Exception e) { - checkConstraintViolation(e, "calculated_field_link_unq_key", "Calculated Field for such entity id is already exists!"); - throw e; - } + log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); + return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java deleted file mode 100644 index 980fff6273..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.cf; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -public class CalculatedFieldConfigUtil { - -// public static CalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { -// if (config == null) { -// return null; -// } -// try { -// CalculatedFieldConfiguration calculatedFieldConfig = new BaseCalculatedFieldConfiguration(); -// Map arguments = new HashMap<>(); -// -// JsonNode argumentsNode = config.get("arguments"); -// if (argumentsNode != null && argumentsNode.isObject()) { -// argumentsNode.fields().forEachRemaining(entry -> { -// String key = entry.getKey(); -// JsonNode argumentNode = entry.getValue(); -// -// CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); -// if (argumentNode.has("entityType") && argumentNode.has("entityId")) { -// String referencedEntityType = argumentNode.get("entityType").asText(); -// UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); -// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); -// } else { -// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); -// } -// argument.setKey(argumentNode.get("key").asText()); -// argument.setType(argumentNode.get("type").asText()); -// -// if (argumentNode.has("defaultValue")) { -// argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); -// } -// -// arguments.put(key, argument); -// }); -// } -// calculatedFieldConfig.setArguments(arguments); -// -// JsonNode outputNode = config.get("output"); -// if (outputNode != null) { -// CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); -// output.setType(outputNode.get("type").asText()); -// output.setExpression(outputNode.get("expression").asText()); -// calculatedFieldConfig.setOutput(output); -// } -// -// return calculatedFieldConfig; -// -// } catch (Exception e) { -// throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); -// } -// } -// -// public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfiguration calculatedFieldConfig, EntityType entityType, UUID entityId) { -// if (calculatedFieldConfig == null) { -// return null; -// } -// try { -// ObjectNode configNode = JacksonUtil.newObjectNode(); -// -// ObjectNode argumentsNode = configNode.putObject("arguments"); -// calculatedFieldConfig.getArguments().forEach((key, argument) -> { -// ObjectNode argumentNode = argumentsNode.putObject(key); -// EntityId referencedEntityId = argument.getEntityId(); -// if (referencedEntityId != null) { -// argumentNode.put("entityType", referencedEntityId.getEntityType().name()); -// argumentNode.put("entityId", referencedEntityId.getId().toString()); -// } else { -// argumentNode.put("entityType", entityType.name()); -// argumentNode.put("entityId", entityId.toString()); -// } -// argumentNode.put("key", argument.getKey()); -// argumentNode.put("type", argument.getType()); -// argumentNode.put("defaultValue", argument.getDefaultValue()); -// }); -// -// if (calculatedFieldConfig.getOutput() != null) { -// ObjectNode outputNode = configNode.putObject("output"); -// outputNode.put("type", calculatedFieldConfig.getOutput().getType()); -// outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); -// } -// -// return configNode; -// -// } catch (Exception e) { -// throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); -// } -// } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 9cb4f08640..725ddd7bef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -117,8 +117,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - String type = config.get("type").asText(); + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { switch (type) { case "SIMPLE": return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 3dc08d6bf1..0ac5a9bc94 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -86,7 +86,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity calculatedFieldService.saveCalculatedFieldLink(tenantId, calculatedFieldLink)) - .isInstanceOf(DataValidationException.class) - .hasMessage("Calculated Field for such entity id is already exists!"); - } - private CalculatedField saveValidCalculatedField() { Device device = createTestDevice(); CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); @@ -148,10 +140,10 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setEntityId(entityId); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); -// calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } @@ -160,22 +152,28 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); calculatedFieldLink.setTenantId(tenantId); calculatedFieldLink.setEntityId(calculatedField.getEntityId()); -// calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); + calculatedFieldLink.setConfiguration(getCalculatedFieldLinkConfiguration()); calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); return calculatedFieldLink; } - private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { - CalculatedFieldConfig config = new CalculatedFieldConfig(); + private CalculatedFieldLinkConfiguration getCalculatedFieldLinkConfiguration() { + CalculatedFieldLinkConfiguration calculatedFieldLinkConfiguration = new CalculatedFieldLinkConfiguration(); + calculatedFieldLinkConfiguration.setTimeSeries(List.of("temperature")); + return calculatedFieldLinkConfiguration; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 3f990b9dd7..0e2d9b797b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -369,25 +369,25 @@ public class CustomerServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setEntityId(savedAsset.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); config.setOutput(output); -// calculatedField.setConfiguration(config); + calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 90d04d5216..33b80c0d99 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,7 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -1212,25 +1212,25 @@ public class DeviceServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setEntityId(deviceWithCf.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(device.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); config.setOutput(output); -// calculatedField.setConfiguration(config); + calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); @@ -1241,5 +1241,4 @@ public class DeviceServiceTest extends AbstractServiceTest { calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } - } From b6b7af8003d7737c8cc5f818c85711b338329db9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 12:51:29 +0200 Subject: [PATCH 019/416] refactored tests --- .../CalculatedFieldControllerTest.java | 15 ++++++++------- .../service/CalculatedFieldServiceTest.java | 18 ------------------ 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 3f505c78ef..67e7bc64bd 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -22,7 +22,8 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; @@ -123,25 +124,25 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(deviceId); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); -// calculatedField.setConfiguration(getCalculatedFieldConfig(null)); + calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } - private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { - CalculatedFieldConfig config = new CalculatedFieldConfig(); + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index de6836900d..a94eb83fa3 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -25,8 +25,6 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -34,7 +32,6 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; -import java.util.List; import java.util.Map; import java.util.UUID; @@ -148,21 +145,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { return calculatedField; } - private CalculatedFieldLink getCalculatedFieldLink(CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); - calculatedFieldLink.setTenantId(tenantId); - calculatedFieldLink.setEntityId(calculatedField.getEntityId()); - calculatedFieldLink.setConfiguration(getCalculatedFieldLinkConfiguration()); - calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); - return calculatedFieldLink; - } - - private CalculatedFieldLinkConfiguration getCalculatedFieldLinkConfiguration() { - CalculatedFieldLinkConfiguration calculatedFieldLinkConfiguration = new CalculatedFieldLinkConfiguration(); - calculatedFieldLinkConfiguration.setTimeSeries(List.of("temperature")); - return calculatedFieldLinkConfiguration; - } - private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); From 23009c9aafd7e1dd616773621284f33abc537616 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 17:21:03 +0200 Subject: [PATCH 020/416] refactored defaultTbCalculatedFieldService methods --- .../entitiy/AbstractTbEntityService.java | 7 + .../entitiy/cf/CalculatedFieldCtx.java | 13 +- .../cf/DefaultTbCalculatedFieldService.java | 177 +++++++----------- .../data/cf/CalculatedFieldConfiguration.java | 3 + 4 files changed, 90 insertions(+), 110 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index d2b3890c70..cef32b9065 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -89,18 +89,25 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; @Autowired + @Lazy protected AccessControlService accessControlService; @Autowired + @Lazy protected TenantService tenantService; @Autowired + @Lazy protected AssetService assetService; @Autowired + @Lazy protected DeviceService deviceService; @Autowired + @Lazy protected AssetProfileService assetProfileService; @Autowired + @Lazy protected DeviceProfileService deviceProfileService; @Autowired + @Lazy protected EntityService entityService; protected boolean isTestProfile() { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 61f28304f2..0a7a5c8264 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -15,17 +15,20 @@ */ package org.thingsboard.server.service.entitiy.cf; -import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @Data -@Builder public class CalculatedFieldCtx { - private final CalculatedFieldId calculatedFieldId; - private final EntityId entityId; - private final CalculatedFieldState state; + private CalculatedFieldId calculatedFieldId; + private EntityId entityId; + private CalculatedFieldState state; + public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { + this.calculatedFieldId = calculatedFieldId; + this.entityId = entityId; + this.state = state; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index e222259a53..b439d8e641 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -38,7 +39,6 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -48,8 +48,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.attributes.AttributesService; @@ -62,6 +61,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -71,7 +71,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -128,98 +127,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private ListenableFuture> fetchAttributesForEntity(TenantId tenantId, EntityId entityId, List keys) { - return attributesService.find(tenantId, entityId, AttributeScope.SERVER_SCOPE, keys); - } - - private ListenableFuture> fetchTimeSeries(TenantId tenantId, EntityId entityId, List keys) { - return timeseriesService.findLatest(tenantId, entityId, keys); - } - - private ListenableFuture initializeStateFromFutures(TenantId tenantId, EntityId entityId, CalculatedField calculatedField, List attributeKeys, List timeSeriesKeys) { - ListenableFuture> attributesFuture = fetchAttributesForEntity(tenantId, entityId, attributeKeys); - ListenableFuture> timeSeriesFuture = fetchTimeSeries(tenantId, entityId, timeSeriesKeys); - - ListenableFuture> combinedFuture = Futures.allAsList(attributesFuture, timeSeriesFuture); - - return Futures.transform(combinedFuture, results -> { - List attributes = (List) results.get(0); - List timeSeries = (List) results.get(1); - - initializeState(calculatedField, attributes, timeSeries); - - return null; - }, MoreExecutors.directExecutor()); - } - - private void initializeState(CalculatedField calculatedField, List attributes, List timeSeries) { - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), - ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); - - CalculatedFieldState state = calculatedFieldCtx.getState(); - - if (state != null) { - String calculation = performCalculation(state.getArguments()); - - Map updatedArguments = state.getArguments(); - - state = CalculatedFieldState.builder() - .arguments(updatedArguments) - .result(calculation) - .build(); - } else { - // initial calculation - Map arguments = calculatedField.getConfiguration().getArguments(); - - Map argumentValues = arguments.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> resolveArgumentValue(entry.getKey(), entry.getValue(), attributes, timeSeries) - )); - - String calculation = performCalculation(argumentValues); - - state = CalculatedFieldState.builder() - .arguments(argumentValues) - .result(calculation) - .build(); - } - - calculatedFieldCtx = new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), state); - states.put(calculatedField.getId(), calculatedFieldCtx); - } - - private String resolveArgumentValue(String key, BaseCalculatedFieldConfiguration.Argument argument, - List attributes, List timeSeries) { - String type = argument.getType(); - String value = null; - - if ("ATTRIBUTES".equals(type)) { - value = attributes.stream() - .filter(attribute -> attribute.getKey().equals(key)) - .map(AttributeKvEntry::getValueAsString) - .findFirst() - .orElse(null); - } else if ("TIME_SERIES".equals(type)) { - value = timeSeries.stream() - .filter(tsKvEntry -> tsKvEntry.getKey().equals(key)) - .map(TsKvEntry::getValueAsString) - .findFirst() - .orElse(null); - } - - return value != null ? value : argument.getDefaultValue(); - } - - private void initializeForProfile(TenantId tenantId, List links, CalculatedField cf, Iterable profileIds) { - for (T profileId : profileIds) { - for (CalculatedFieldLink link : links) { - CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, profileId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } - } - @Override public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { try { @@ -232,21 +139,16 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - for (CalculatedFieldLink link : links) { - CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, callback); case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - initializeForProfile(tenantId, links, cf, assetIds); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, callback)); } case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - initializeForProfile(tenantId, links, cf, deviceIds); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -359,7 +261,72 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private String performCalculation(Map argumentValues) { + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, TbCallback callback) { + Map arguments = calculatedField.getConfiguration().getArguments(); + Map argumentValues = new HashMap<>(); + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + @Override + public void onSuccess(Optional result) { + String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); + argumentValues.put(key, value); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to fetch data for type: {}", argument.getType(), t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor)); + + updateOrInitializeState(calculatedField, argumentValues); + + } + + private ListenableFuture> fetchArgumentValue(TenantId tenantId, BaseCalculatedFieldConfiguration.Argument argument) { + return switch (argument.getType()) { + case "ATTRIBUTES" -> Futures.transform( + attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + case "TIME_SERIES" -> Futures.transform( + timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private void updateOrInitializeState(CalculatedField calculatedField, Map argumentValues) { + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state != null) { + // calculation based on the previous data + String calculation = performCalculation(state.getArguments(), calculatedField.getConfiguration()); + + Map updatedArguments = new HashMap<>(state.getArguments()); + + state = CalculatedFieldState.builder() + .arguments(updatedArguments) + .result(calculation) + .build(); + } else { + // initial calculation + String calculation = performCalculation(argumentValues, calculatedField.getConfiguration()); + + state = CalculatedFieldState.builder() + .arguments(argumentValues) + .result(calculation) + .build(); + } + calculatedFieldCtx.setState(state); + states.put(calculatedField.getId(), calculatedFieldCtx); + } + + private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { + BaseCalculatedFieldConfiguration.Output output = calculatedFieldConfiguration.getOutput(); return "calculation"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index deaeebcb50..cfae2e5a0e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -41,6 +41,9 @@ public interface CalculatedFieldConfiguration { Map getArguments(); + @JsonIgnore + BaseCalculatedFieldConfiguration.Output getOutput(); + @JsonIgnore List getReferencedEntities(); From 88e5da7a14068bb90bd2cb11b85217842446c220 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 09:06:12 +0200 Subject: [PATCH 021/416] changed key for states map --- .../cf/DefaultTbCalculatedFieldService.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index b439d8e641..2fe1f65015 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -90,7 +90,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); @Value("${state.initFetchPackSize:50000}") @Getter @@ -139,16 +139,16 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, callback); + case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, entityId, callback); case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); } case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -180,7 +180,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); - states.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); } catch (Exception e) { log.trace("Failed to process calculated field delete msg: [{}]", proto, e); callback.onFailure(e); @@ -261,7 +261,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, TbCallback callback) { + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @@ -278,7 +278,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } }, calculatedFieldCallbackExecutor)); - updateOrInitializeState(calculatedField, argumentValues); + updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -296,8 +296,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void updateOrInitializeState(CalculatedField calculatedField, Map argumentValues) { - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + String ctxId = calculatedField.getId().getId() + "_" + entityId.getId(); + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); CalculatedFieldState state = calculatedFieldCtx.getState(); @@ -322,7 +323,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp .build(); } calculatedFieldCtx.setState(state); - states.put(calculatedField.getId(), calculatedFieldCtx); + states.put(ctxId, calculatedFieldCtx); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { From 4d8b62eb21e214cbbf2940a4ded8bac678eadf3c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 09:55:03 +0200 Subject: [PATCH 022/416] moved logic of cf add/update/delete msg to a single method --- .../cf/DefaultTbCalculatedFieldService.java | 55 +++++++++---------- .../entitiy/cf/TbCalculatedFieldService.java | 6 +- common/proto/src/main/proto/queue.proto | 19 ++----- 3 files changed, 30 insertions(+), 50 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 2fe1f65015..4c83fcf7b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -128,10 +128,17 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } @Override - public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { + public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + if (proto.getDeleted()) { + onCalculatedFieldDelete(calculatedFieldId, callback); + callback.onSuccess(); + } + if (proto.getUpdated()) { + onCalculatedFieldDelete(calculatedFieldId, callback); + } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { @@ -163,30 +170,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - @Override - public void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - } catch (Exception e) { - log.trace("Failed to process calculated field update msg: [{}]", proto, e); - callback.onFailure(e); - } - } - - @Override - public void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback) { - try { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - calculatedFieldLinks.remove(calculatedFieldId); - calculatedFields.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); - } catch (Exception e) { - log.trace("Failed to process calculated field delete msg: [{}]", proto, e); - callback.onFailure(e); - } - } - @Override public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; @@ -223,13 +206,25 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); + } catch (Exception e) { + log.trace("Failed to delete calculated field.", e); + callback.onFailure(e); + } + } + private void fetchCalculatedFields() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); // TODO: read all states(CalculatedFieldCtx) - states.keySet().removeIf(calculatedFieldId -> !calculatedFields.containsKey(calculatedFieldId)); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } private void checkEntityExistence(TenantId tenantId, EntityId entityId) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 4f14270c3d..aa77d29702 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -24,11 +24,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback); - - void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback); - - void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback); + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index e1d6320779..b41e010a2d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1267,25 +1267,14 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } -message CalculatedFieldAddMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; -} - -message CalculatedFieldUpdateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; -} - -message CalculatedFieldDeleteMsgProto { +message CalculatedFieldMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; int64 calculatedFieldIdMSB = 3; int64 calculatedFieldIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; } /** From 3072861a8fa986cda6b57561a343fd3d1c6c094b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 17:42:55 +0200 Subject: [PATCH 023/416] implemented methods for calculated field update/delete in cluster service --- application/pom.xml | 4 + .../entitiy/EntityStateSourcingListener.java | 28 +++++- .../cf/DefaultTbCalculatedFieldService.java | 11 ++- .../service/entitiy/cf/RocksDBService.java | 95 +++++++++++++++++++ .../queue/DefaultTbClusterService.java | 34 ++++++- .../queue/DefaultTbCoreConsumerService.java | 21 +++- .../server/utils/RocksDBConfig.java | 52 ++++++++++ .../src/main/resources/thingsboard.yml | 4 + .../server/cluster/TbClusterService.java | 5 + common/proto/src/main/proto/queue.proto | 1 + .../dao/cf/BaseCalculatedFieldService.java | 7 +- pom.xml | 7 ++ 12 files changed, 260 insertions(+), 9 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java diff --git a/application/pom.xml b/application/pom.xml index 17b179c767..62d2a49908 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -369,6 +369,10 @@ com.google.firebase firebase-admin + + org.rocksdb + rocksdbjni + diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 863be23e42..6c2e995b82 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -118,7 +119,11 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } - default -> {} + case CALCULATED_FIELD -> { + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); + } + default -> { + } } } @@ -130,7 +135,7 @@ public class EntityStateSourcingListener { return; } EntityType entityType = entityId.getEntityType(); - if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); return; } @@ -149,7 +154,8 @@ public class EntityStateSourcingListener { case RULE_CHAIN -> { RuleChain ruleChain = (RuleChain) event.getEntity(); if (RuleChainType.CORE.equals(ruleChain.getType())) { - Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() {}); + Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() { + }); if (referencingRuleChainIds != null) { referencingRuleChainIds.forEach(referencingRuleChainId -> tbClusterService.broadcastEntityStateChangeEvent(tenantId, referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); @@ -177,7 +183,12 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } - default -> {} + case CALCULATED_FIELD -> { + CalculatedField calculatedField = (CalculatedField) event.getEntity(); + tbClusterService.onCalculatedFieldDeleted(tenantId, calculatedField, null); + } + default -> { + } } } @@ -247,6 +258,15 @@ public class EntityStateSourcingListener { } } + private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { + CalculatedField calculatedField = (CalculatedField) entity; + CalculatedField oldCalculatedField = null; + if (oldEntity instanceof CalculatedField) { + oldCalculatedField = (CalculatedField) oldEntity; + } + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 4c83fcf7b5..06c87ff4d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.AttributeScope; @@ -64,6 +65,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Random; import java.util.UUID; @@ -71,6 +73,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -83,6 +86,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final CalculatedFieldService calculatedFieldService; private final AttributesService attributesService; private final TimeseriesService timeseriesService; + private final RocksDBService rocksDBService; private ListeningScheduledExecutorService scheduledExecutor; private ListeningExecutorService calculatedFieldExecutor; @@ -212,6 +216,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); + List statesToRemove = states.keySet().stream() + .filter(key -> key.startsWith(calculatedFieldId.getId().toString())) + .collect(Collectors.toList()); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to delete calculated field.", e); callback.onFailure(e); @@ -223,7 +231,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - // TODO: read all states(CalculatedFieldCtx) + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.convertValue(ctx, CalculatedFieldCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } @@ -319,6 +327,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } calculatedFieldCtx.setState(state); states.put(ctxId, calculatedFieldCtx); + rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.toString(calculatedFieldCtx))); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java new file mode 100644 index 0000000000..2ba3e9bb4f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; +import org.springframework.stereotype.Service; +import org.thingsboard.server.utils.RocksDBConfig; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +@Slf4j +public class RocksDBService { + + private final RocksDB db; + private final WriteOptions writeOptions; + + public RocksDBService(RocksDBConfig config) throws RocksDBException { + this.db = config.getDb(); + this.writeOptions = new WriteOptions().setSync(true); + } + + public void put(String key, String value) { + try { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.error("Failed to store data to RocksDB", e); + } + } + + public void delete(String key) { + try { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.error("Failed to delete data from RocksDB", e); + } + } + + public void deleteAll(List keys) { + try (WriteBatch batch = new WriteBatch()) { + for (String key : keys) { + batch.delete(key.getBytes(StandardCharsets.UTF_8)); + } + db.write(writeOptions, batch); + } catch (RocksDBException e) { + log.error("Failed to delete data from RocksDB", e); + } + } + + public String get(String key) { + try { + byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); + return value != null ? new String(value, StandardCharsets.UTF_8) : null; + } catch (RocksDBException e) { + log.error("Failed to retrieve data from RocksDB", e); + return null; + } + } + + public Map getAll() { + Map map = new HashMap<>(); + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + String value = new String(iterator.value(), StandardCharsets.UTF_8); + map.put(key, value); + } + } catch (Exception e) { + log.error("Failed to retrieve data from RocksDB", e); + } + return map; + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 301f8e8838..f7c8230716 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,10 +38,12 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -666,7 +668,8 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> + pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); @@ -743,4 +746,33 @@ public class DefaultTbClusterService implements TbClusterService { } } + @Override + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { + var created = oldCalculatedField == null; + broadcastEntityChangeToTransport(calculatedField.getTenantId(), calculatedField.getId(), calculatedField, null); + broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + sendCalculatedFieldEvent(calculatedField.getTenantId(), calculatedField.getId(), created, !created, false); + } + + @Override + public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); + sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); + broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + } + + private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { + TransportProtos.CalculatedFieldMsgProto.Builder builder = TransportProtos.CalculatedFieldMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()); + builder.setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); + builder.setAdded(added); + builder.setUpdated(updated); + builder.setDeleted(deleted); + TransportProtos.CalculatedFieldMsgProto msg = builder.build(); + pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2a64dd3388..979c419003 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; @@ -85,6 +86,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -148,6 +150,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; @@ -175,7 +178,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java new file mode 100644 index 0000000000..75a4dd138a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import jakarta.annotation.PreDestroy; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class RocksDBConfig { + + @Value("${rocksdb.db_path}") + private String dbPath; + private RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + public RocksDB getDb() throws RocksDBException { + if (db == null) { + Options options = new Options().setCreateIfMissing(true); + db = RocksDB.open(options, dbPath); + } + return db; + } + + @PreDestroy + public void close() { + if (db != null) { + db.close(); + db = null; + } + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 106810a7a1..8b1aeedec8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,6 +424,10 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls +rocksdb: + # Rocksdb path + db_path: "${ROCKS_DB_PATH:}" + # Actor system parameters actors: system: diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 5112e93da7..f173005107 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; @@ -114,4 +115,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField); + + void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback); + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b41e010a2d..95e4fa8601 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1512,6 +1512,7 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; + CalculatedFieldMsgProto calculatedFieldMsg = 53; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index db5539eb96..8603dadcc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -29,20 +29,21 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @Service("CalculatedFieldDaoService") @Slf4j @RequiredArgsConstructor -public class BaseCalculatedFieldService implements CalculatedFieldService { +public class BaseCalculatedFieldService extends AbstractEntityService implements CalculatedFieldService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; @@ -60,6 +61,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) + .entity(savedCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; } catch (Exception e) { checkConstraintViolation(e, diff --git a/pom.xml b/pom.xml index a1d5f26ecf..9b835516fc 100755 --- a/pom.xml +++ b/pom.xml @@ -166,6 +166,8 @@ 1.6.1 2.19.0 9.2.0 + + 9.4.0 @@ -2272,6 +2274,11 @@ metadata-extractor ${drewnoakes-metadata-extractor.version} + + org.rocksdb + rocksdbjni + ${rocksdbjni.version} + From f75eaf8226cff33ad0d8f935f1750394a3475375 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 14 Nov 2024 10:23:57 +0200 Subject: [PATCH 024/416] added rocksdb path --- .../entitiy/AbstractTbEntityService.java | 7 ------- .../server/utils/RocksDBConfig.java | 2 +- .../src/main/resources/thingsboard.yml | 2 +- .../alarm/DefaultTbAlarmServiceTest.java | 21 +++++++++++++++++++ .../DefaultTbAlarmCommentServiceTest.java | 21 +++++++++++++++++++ 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index cef32b9065..d2b3890c70 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -89,25 +89,18 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; @Autowired - @Lazy protected AccessControlService accessControlService; @Autowired - @Lazy protected TenantService tenantService; @Autowired - @Lazy protected AssetService assetService; @Autowired - @Lazy protected DeviceService deviceService; @Autowired - @Lazy protected AssetProfileService assetProfileService; @Autowired - @Lazy protected DeviceProfileService deviceProfileService; @Autowired - @Lazy protected EntityService entityService; protected boolean isTestProfile() { diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java index 75a4dd138a..9c3c02f472 100644 --- a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java +++ b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java @@ -25,7 +25,7 @@ import org.springframework.stereotype.Component; @Component public class RocksDBConfig { - @Value("${rocksdb.db_path}") + @Value("${rocksdb.db_path:${java.io.tmpdir}/rocksdb}") private String dbPath; private RocksDB db; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8b1aeedec8..97405f35d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -426,7 +426,7 @@ sql: rocksdb: # Rocksdb path - db_path: "${ROCKS_DB_PATH:}" + db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" # Actor system parameters actors: diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index 60a0ac3bf6..bc1726ff41 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -38,10 +38,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -81,6 +88,20 @@ public class DefaultTbAlarmServiceTest { protected TbClusterService tbClusterService; @MockBean private EntitiesVersionControlService vcService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmService service; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java index b36ca51416..71c295edc9 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java @@ -35,10 +35,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.entitiy.alarm.DefaultTbAlarmCommentService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import java.util.UUID; @@ -72,6 +79,20 @@ public class DefaultTbAlarmCommentServiceTest { protected CustomerService customerService; @MockBean protected TbClusterService tbClusterService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmCommentService service; From 8079ef66efc3be7f54305758390c296cd6848356 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 14 Nov 2024 14:44:40 +0200 Subject: [PATCH 025/416] added logs --- .../entitiy/cf/CalculatedFieldCtx.java | 3 ++ .../entitiy/cf/CalculatedFieldState.java | 1 + .../cf/DefaultTbCalculatedFieldService.java | 29 ++++++++++++------- .../data/cf/CalculatedFieldConfiguration.java | 1 - .../dao/model/sql/CalculatedFieldEntity.java | 8 ++--- .../model/sql/CalculatedFieldLinkEntity.java | 4 +-- ...efaultNativeCalculatedFieldRepository.java | 4 +-- 7 files changed, 31 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 0a7a5c8264..021a9bbc83 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -26,6 +26,9 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldState state; + public CalculatedFieldCtx() { + } + public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { this.calculatedFieldId = calculatedFieldId; this.entityId = entityId; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index dc07b820ea..f84f4a57c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -24,6 +24,7 @@ import java.util.Map; @Builder public class CalculatedFieldState { + // TODO: use value object(TsKv) instead of string Map arguments; String result; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 06c87ff4d1..51ff540a64 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -67,7 +67,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -113,7 +112,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, new Random().nextInt(defaultCalculatedFieldCheckIntervalInSec), defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); + scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, 0, defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); } @PreDestroy @@ -136,11 +135,15 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { + log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } if (proto.getUpdated()) { + log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + //TODO: improve the check. Maybe it was renamed or just the name of the result changed. onCalculatedFieldDelete(calculatedFieldId, callback); } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); @@ -150,13 +153,18 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, entityId, callback); + case ASSET, DEVICE -> { + log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntity(tenantId, cf, entityId, callback); + } case ASSET_PROFILE -> { + log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); } case DEVICE_PROFILE -> { + log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); @@ -164,12 +172,14 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } else { - //Calculated field or entity was probably deleted while message was in queue; + //Calculated field was probably deleted while message was in queue; + log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); callback.onSuccess(); } } catch (Exception e) { - log.trace("Failed to process calculated field add msg: [{}]", proto, e); + log.trace("Failed to process calculated field msg: [{}]", proto, e); callback.onFailure(e); } } @@ -210,7 +220,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { calculatedFieldLinks.remove(calculatedFieldId); @@ -221,7 +230,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp .collect(Collectors.toList()); rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { - log.trace("Failed to delete calculated field.", e); + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); } } @@ -231,7 +240,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.convertValue(ctx, CalculatedFieldCtx.class))); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } @@ -276,7 +285,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp @Override public void onFailure(Throwable t) { - log.warn("Failed to fetch data for type: {}", argument.getType(), t); + log.warn("Failed to initialize state for entity: [{}]", entityId, t); callback.onFailure(t); } }, calculatedFieldCallbackExecutor)); @@ -327,7 +336,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } calculatedFieldCtx.setState(state); states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.toString(calculatedFieldCtx))); + rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index cfae2e5a0e..7e6a77f006 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -41,7 +41,6 @@ public interface CalculatedFieldConfiguration { Map getArguments(); - @JsonIgnore BaseCalculatedFieldConfiguration.Output getOutput(); @JsonIgnore diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 725ddd7bef..efb262477b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -56,7 +56,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem private UUID tenantId; @Column(name = CALCULATED_FIELD_ENTITY_TYPE) - private EntityType entityType; + private String entityType; @Column(name = CALCULATED_FIELD_ENTITY_ID) private UUID entityId; @@ -88,12 +88,12 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.setUuid(calculatedField.getUuidId()); this.createdTime = calculatedField.getCreatedTime(); this.tenantId = calculatedField.getTenantId().getId(); - this.entityType = calculatedField.getEntityId().getEntityType(); + this.entityType = calculatedField.getEntityId().getEntityType().name(); this.entityId = calculatedField.getEntityId().getId(); this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(entityType, entityId); + this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -109,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 0ac5a9bc94..b4d6677be4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -53,7 +53,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity Date: Thu, 14 Nov 2024 17:19:14 +0200 Subject: [PATCH 026/416] added logic to handle cf update msg --- .../entitiy/cf/CalculatedFieldState.java | 24 ++++--- .../cf/DefaultTbCalculatedFieldService.java | 67 ++++++++++++------- .../cf/SimpleCalculatedFieldState.java | 49 ++++++++++++++ .../CalculatedFieldControllerTest.java | 3 +- .../cf/BaseCalculatedFieldConfiguration.java | 1 + .../common/data/cf/CalculatedField.java | 12 ++-- .../data/cf/CalculatedFieldConfiguration.java | 2 +- .../common/data/cf/CalculatedFieldType.java | 22 ++++++ .../SimpleCalculatedFieldConfiguration.java | 4 +- .../dao/model/sql/CalculatedFieldEntity.java | 9 +-- ...efaultNativeCalculatedFieldRepository.java | 7 +- .../server/dao/service/AssetServiceTest.java | 3 +- .../service/CalculatedFieldServiceTest.java | 3 +- .../dao/service/CustomerServiceTest.java | 3 +- .../server/dao/service/DeviceServiceTest.java | 3 +- 15 files changed, 160 insertions(+), 52 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index f84f4a57c2..9fb6a009fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -15,17 +15,25 @@ */ package org.thingsboard.server.service.entitiy.cf; -import lombok.Builder; -import lombok.Data; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import java.util.Map; -@Data -@Builder -public class CalculatedFieldState { +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE") +}) +public interface CalculatedFieldState { - // TODO: use value object(TsKv) instead of string - Map arguments; - String result; + CalculatedFieldType getType(); + + void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 51ff540a64..49c64a1262 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -141,12 +142,11 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - //TODO: improve the check. Maybe it was renamed or just the name of the result changed. - onCalculatedFieldDelete(calculatedFieldId, callback); + onCalculatedFieldUpdate(cf, callback); } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { EntityId entityId = cf.getEntityId(); @@ -235,6 +235,31 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + if (hasSignificantChanged(oldCalculatedField, newCalculatedField)) { + onCalculatedFieldDelete(newCalculatedField.getId(), callback); + } else { + calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + callback.onSuccess(); + } + } + + private boolean hasSignificantChanged(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean outputNameChanged = !oldConfig.getOutput().getName().equals(newConfig.getOutput().getName()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputNameChanged; + } + private void fetchCalculatedFields() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); @@ -309,39 +334,33 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - String ctxId = calculatedField.getId().getId() + "_" + entityId.getId(); + String ctxId = generateCtxId(calculatedField.getId(), entityId); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); CalculatedFieldState state = calculatedFieldCtx.getState(); - if (state != null) { - // calculation based on the previous data - String calculation = performCalculation(state.getArguments(), calculatedField.getConfiguration()); - - Map updatedArguments = new HashMap<>(state.getArguments()); - - state = CalculatedFieldState.builder() - .arguments(updatedArguments) - .result(calculation) - .build(); + state.performCalculation(argumentValues, calculatedField.getConfiguration(), false); } else { - // initial calculation - String calculation = performCalculation(argumentValues, calculatedField.getConfiguration()); - - state = CalculatedFieldState.builder() - .arguments(argumentValues) - .result(calculation) - .build(); + CalculatedFieldState newState = createStateByType(calculatedField.getType()); + newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); } calculatedFieldCtx.setState(state); + states.put(ctxId, calculatedFieldCtx); rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); } - private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { - BaseCalculatedFieldConfiguration.Output output = calculatedFieldConfiguration.getOutput(); - return "calculation"; + private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { + return switch (calculatedFieldType) { + case SIMPLE -> new SimpleCalculatedFieldState(); + default -> + throw new IllegalArgumentException("Invalid calculated field type '" + calculatedFieldType + "'."); + }; + } + + private String generateCtxId(CalculatedFieldId calculatedFieldId, EntityId entityId) { + return calculatedFieldId.getId() + "_" + entityId.getId(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java new file mode 100644 index 0000000000..8a90893262 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class SimpleCalculatedFieldState implements CalculatedFieldState { + + // TODO: use value object(TsKv) instead of string + Map arguments = new HashMap<>(); + String result; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } + + @Override + public void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation) { + if (initialCalculation) { + // todo: perform initial calculation + this.arguments = argumentValues; + } else { + // todo: perform calculation based on previous data + this.arguments.putAll(argumentValues); + } + this.result = "result"; + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 67e7bc64bd..ebda2cbf82 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -124,7 +125,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(deviceId); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(null)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 3894835abc..4575e414ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -115,6 +115,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel @Data public static class Output { + private String name; private String type; private String expression; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index d96de37a39..ceb1222fe2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -20,15 +20,17 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; -import java.io.Serializable; - @Schema @Data @EqualsAndHashCode(callSuper = true) @@ -41,7 +43,7 @@ public class CalculatedField extends BaseData implements HasN @NoXss @Length(fieldName = "type") - private String type; + private CalculatedFieldType type; @NoXss @Length(fieldName = "name") @Schema(description = "User defined name of the calculated field.") @@ -65,7 +67,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index 7e6a77f006..f733c35310 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -37,7 +37,7 @@ import java.util.UUID; public interface CalculatedFieldConfiguration { @JsonIgnore - String getType(); + CalculatedFieldType getType(); Map getArguments(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java new file mode 100644 index 0000000000..89173b35b9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +public enum CalculatedFieldType { + + SIMPLE, SCRIPT + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java index d635e0d82e..327f9cdc75 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java @@ -33,7 +33,7 @@ public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfi } @Override - public String getType() { - return "SIMPLE"; + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index efb262477b..3c45a81cb5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -25,6 +25,7 @@ import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -90,7 +91,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.tenantId = calculatedField.getTenantId().getId(); this.entityType = calculatedField.getEntityId().getEntityType().name(); this.entityId = calculatedField.getEntityId().getId(); - this.type = calculatedField.getType(); + this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); @@ -106,7 +107,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setCreatedTime(createdTime); calculatedField.setTenantId(TenantId.fromUUID(tenantId)); calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - calculatedField.setType(type); + calculatedField.setType(CalculatedFieldType.valueOf(type)); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); @@ -118,8 +119,8 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - switch (type) { - case "SIMPLE": + switch (CalculatedFieldType.valueOf(type)) { + case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 73f334e8f1..417a468b2c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -73,7 +74,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF UUID tenantId = (UUID) row.get("tenant_id"); EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); UUID entityId = (UUID) row.get("entity_id"); - String type = (String) row.get("type"); + CalculatedFieldType type = CalculatedFieldType.valueOf((String) row.get("type")); String name = (String) row.get("name"); int configurationVersion = (int) row.get("configuration_version"); JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); @@ -133,9 +134,9 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF }); } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(String type, JsonNode config, EntityType entityType, UUID entityId) { + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { switch (type) { - case "SIMPLE": + case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 432d559598..43b51ce2ac 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -874,7 +875,7 @@ public class AssetServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(savedAssetWithCf.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index a94eb83fa3..d5d025a46d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -25,6 +25,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -137,7 +138,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setEntityId(entityId); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 0e2d9b797b..1c5e0d8f49 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -369,7 +370,7 @@ public class CustomerServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(savedAsset.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 33b80c0d99..8f1f16e631 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1212,7 +1213,7 @@ public class DeviceServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(deviceWithCf.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); From 7c433d47e2db2b291ebd99d2047f3c2e10887178 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 08:46:35 +0200 Subject: [PATCH 027/416] changed type value in cf data validator test --- .../service/validator/CalculatedFieldDataValidatorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java index 6d0eb0ad38..10df39e04b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -20,6 +20,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -44,7 +45,7 @@ public class CalculatedFieldDataValidatorTest { @Test public void testUpdateNonExistingCalculatedField() { CalculatedField calculatedField = new CalculatedField(CALCULATED_FIELD_ID); - calculatedField.setType("Simple"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test"); given(calculatedFieldDao.findById(TENANT_ID, CALCULATED_FIELD_ID.getId())).willReturn(null); From 2fc32ee232c37cb9201e18adbebcc4d5e5ab7b24 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 12:07:41 +0200 Subject: [PATCH 028/416] removed unnecessary methods --- .../entitiy/cf/CalculatedFieldState.java | 2 + .../cf/DefaultTbCalculatedFieldService.java | 24 +++++--- .../src/main/resources/thingsboard.yml | 4 ++ .../server/dao/cf/CalculatedFieldService.java | 9 ++- .../dao/cf/BaseCalculatedFieldService.java | 55 ++++++++++++++++--- .../server/dao/cf/CalculatedFieldDao.java | 2 - .../server/dao/cf/CalculatedFieldLinkDao.java | 3 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 7 --- .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 7 ++- 9 files changed, 84 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index 9fb6a009fe..221c44b94c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; @@ -32,6 +33,7 @@ import java.util.Map; }) public interface CalculatedFieldState { + @JsonIgnore CalculatedFieldType getType(); void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 49c64a1262..f2f16de844 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -96,7 +96,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); - @Value("${state.initFetchPackSize:50000}") + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -145,7 +145,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldUpdate(cf, callback); + boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); + if (!shouldReinit) { + return; + } } List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { @@ -172,12 +175,13 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } else { //Calculated field was probably deleted while message was in queue; log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); callback.onSuccess(); } + callback.onSuccess(); + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to process calculated field msg: [{}]", proto, e); callback.onFailure(e); @@ -235,17 +239,20 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); - if (hasSignificantChanged(oldCalculatedField, newCalculatedField)) { + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { onCalculatedFieldDelete(newCalculatedField.getId(), callback); } else { calculatedFields.put(newCalculatedField.getId(), newCalculatedField); callback.onSuccess(); + shouldReinit = false; } + return shouldReinit; } - private boolean hasSignificantChanged(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; } @@ -255,9 +262,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputNameChanged = !oldConfig.getOutput().getName().equals(newConfig.getOutput().getName()); + boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputNameChanged; + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; } private void fetchCalculatedFields() { @@ -344,6 +351,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } else { CalculatedFieldState newState = createStateByType(calculatedField.getType()); newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); + state = newState; } calculatedFieldCtx.setState(state); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 97405f35d3..699d2012cc 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,6 +424,10 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls +# Calculated Field parameters +calculatedField: + initFetchPackSize: "${INIT_FETCH_PACK_SIZE:50000}" + rocksdb: # Rocksdb path db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index c12acade6c..4bb67f9f0d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -33,6 +34,8 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); @@ -45,13 +48,15 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + List findAllCalculatedFieldLinks(); List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - PageData findAllCalculatedFieldLinks(PageLink pageLink); + ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); - boolean existsByEntityId(TenantId tenantId, EntityId entityId); + PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 8603dadcc9..fd669f7865 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -30,7 +31,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; @@ -55,14 +58,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public CalculatedField save(CalculatedField calculatedField) { - calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); + CalculatedField oldCalculatedField = calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); try { TenantId tenantId = calculatedField.getTenantId(); log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) - .entity(savedCalculatedField).created(calculatedField.getId() == null).build()); + .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; } catch (Exception e) { checkConstraintViolation(e, @@ -80,6 +83,13 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); } + @Override + public ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findCalculatedFieldByIdAsync [{}]", calculatedFieldId); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + return calculatedFieldDao.findByIdAsync(tenantId, calculatedFieldId.getId()); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); @@ -95,10 +105,28 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); - calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); + deleteEntity(tenantId, calculatedFieldId, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + CalculatedField calculatedField = calculatedFieldDao.findById(tenantId, id.getId()); + if (calculatedField == null) { + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent calculated field."); + } + } + deleteCalculatedField(tenantId, calculatedField); + } + + private void deleteCalculatedField(TenantId tenantId, CalculatedField calculatedField) { + log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedField.getId()); + calculatedFieldDao.removeById(tenantId, calculatedField.getUuidId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(calculatedField.getId()).entity(calculatedField).build()); } @Override @@ -125,6 +153,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); } + @Override + public ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { + log.trace("Executing findCalculatedFieldLinkByIdAsync [{}]", calculatedFieldLinkId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); + return calculatedFieldLinkDao.findByIdAsync(tenantId, calculatedFieldLinkId.getId()); + } + @Override public List findAllCalculatedFieldLinks() { log.trace("Executing findAllCalculatedFieldLinks"); @@ -137,6 +173,12 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); } + @Override + public ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksByIdAsync, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldIdAsync(tenantId, calculatedFieldId); + } + @Override public PageData findAllCalculatedFieldLinks(PageLink pageLink) { log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); @@ -144,11 +186,6 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findAll(pageLink); } - @Override - public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); - } - @Override public boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId) { return calculatedFieldDao.findAllByTenantId(tenantId).stream() diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 4abe02a09b..d1c9fd86bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -26,8 +26,6 @@ import java.util.List; public interface CalculatedFieldDao extends Dao { - boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); - List findAllByTenantId(TenantId tenantId); List findAll(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 728e19b890..34f2129bd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; @@ -28,6 +29,8 @@ public interface CalculatedFieldLinkDao extends Dao { List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 1137b91947..bdc701070d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; @@ -31,7 +30,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; @@ -46,11 +44,6 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId) { return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantId(tenantId.getId())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index a2f8f224c1..417b529dc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.cf; +import com.google.common.util.concurrent.ListenableFuture; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; @@ -22,7 +23,6 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +49,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return service.submit(() -> findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId)); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); From 24281b48b5f3b720a5b963322c474a9346a97f77 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 17:11:09 +0200 Subject: [PATCH 029/416] removed properrty from thingsboard.yml --- application/src/main/resources/thingsboard.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 699d2012cc..97405f35d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,10 +424,6 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls -# Calculated Field parameters -calculatedField: - initFetchPackSize: "${INIT_FETCH_PACK_SIZE:50000}" - rocksdb: # Rocksdb path db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" From c39a373038c300a306dc2cf87bcfeebd898e4073 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 19 Nov 2024 17:21:25 +0200 Subject: [PATCH 030/416] added calculated field execution service --- .../cf/CalculatedFieldExecutionService.java | 25 ++ ...efaultCalculatedFieldExecutionService.java | 309 ++++++++++++++++++ .../entitiy/EntityStateSourcingListener.java | 16 +- .../entitiy/cf/CalculatedFieldCtx.java | 9 +- .../entitiy/cf/CalculatedFieldCtxId.java | 21 ++ .../entitiy/cf/CalculatedFieldResult.java | 46 +++ .../entitiy/cf/CalculatedFieldState.java | 9 +- .../cf/DefaultTbCalculatedFieldService.java | 258 --------------- .../service/entitiy/cf/RocksDBService.java | 2 + .../cf/ScriptCalculatedFieldState.java | 48 +++ .../cf/SimpleCalculatedFieldState.java | 37 ++- .../entitiy/cf/TbCalculatedFieldService.java | 4 - .../queue/DefaultTbCoreConsumerService.java | 10 +- .../CalculatedFieldControllerTest.java | 3 +- .../server/dao/cf/CalculatedFieldService.java | 2 + .../server/common/data/cf/Argument.java | 32 ++ .../cf/BaseCalculatedFieldConfiguration.java | 8 - .../data/cf/CalculatedFieldConfiguration.java | 2 +- .../dao/cf/BaseCalculatedFieldService.java | 5 + .../server/dao/cf/CalculatedFieldDao.java | 2 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 + .../server/dao/service/AssetServiceTest.java | 3 +- .../service/CalculatedFieldServiceTest.java | 3 +- .../dao/service/CustomerServiceTest.java | 3 +- .../server/dao/service/DeviceServiceTest.java | 3 +- 25 files changed, 566 insertions(+), 299 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java new file mode 100644 index 0000000000..6daa88d771 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; + +public interface CalculatedFieldExecutionService { + + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java new file mode 100644 index 0000000000..c0eeec02bb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -0,0 +1,309 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtx; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtxId; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldState; +import org.thingsboard.server.service.entitiy.cf.RocksDBService; +import org.thingsboard.server.service.entitiy.cf.ScriptCalculatedFieldState; +import org.thingsboard.server.service.entitiy.cf.SimpleCalculatedFieldState; +import org.thingsboard.server.service.partition.AbstractPartitionBasedService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@TbCoreComponent +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { + + private final CalculatedFieldService calculatedFieldService; + private final AssetService assetService; + private final DeviceService deviceService; + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private final RocksDBService rocksDBService; + + private ListeningExecutorService calculatedFieldExecutor; + private ListeningExecutorService calculatedFieldCallbackExecutor; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @PostConstruct + public void init() { + super.init(); + calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(this::fetchCalculatedFields); + } + + @PreDestroy + public void stop() { + if (calculatedFieldExecutor != null) { + calculatedFieldExecutor.shutdownNow(); + } + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + + @Override + protected String getServiceName() { + return "Calculated Field Execution"; + } + + @Override + protected String getSchedulerExecutorName() { + return "calculated-field-scheduled"; + } + + @Override + protected Map>> onAddedPartitions(Set addedPartitions) { + // TODO: implementation for cluster mode + return Map.of(); + } + + @Override + protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { + // TODO: implementation for cluster mode + } + + @Override + public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); + if (proto.getDeleted()) { + log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); + onCalculatedFieldDelete(calculatedFieldId, callback); + callback.onSuccess(); + } + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + if (proto.getUpdated()) { + log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); + if (!shouldReinit) { + return; + } + } + List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cf != null) { + EntityId entityId = cf.getEntityId(); + calculatedFields.put(calculatedFieldId, cf); + calculatedFieldLinks.put(calculatedFieldId, links); + switch (entityId.getEntityType()) { + case ASSET, DEVICE -> { + log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntity(tenantId, cf, entityId, callback); + } + case ASSET_PROFILE -> { + log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); + PageDataIterable assetIds = new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); + } + case DEVICE_PROFILE -> { + log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); + PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); + } + default -> + throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + } + } else { + //Calculated field was probably deleted while message was in queue; + log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); + callback.onSuccess(); + } + callback.onSuccess(); + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to process calculated field msg: [{}]", proto, e); + callback.onFailure(e); + } + } + + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { + onCalculatedFieldDelete(newCalculatedField.getId(), callback); + } else { + calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + callback.onSuccess(); + shouldReinit = false; + } + return shouldReinit; + } + + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + List statesToRemove = states.keySet().stream() + .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) + .map(JacksonUtil::writeValueAsString) + .toList(); + rocksDBService.deleteAll(statesToRemove); + } catch (Exception e) { + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); + callback.onFailure(e); + } + } + + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; + } + + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + } + + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { + Map arguments = calculatedField.getConfiguration().getArguments(); + Map argumentValues = new HashMap<>(); + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + @Override + public void onSuccess(Optional result) { + String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); + argumentValues.put(key, value); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to initialize state for entity: [{}]", entityId, t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor)); + + updateOrInitializeState(calculatedField, entityId, argumentValues); + + } + + private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { + return switch (argument.getType()) { + case "ATTRIBUTES" -> Futures.transform( + attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + case "TIME_SERIES" -> Futures.transform( + timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state == null) { + state = createStateByType(calculatedField.getType()); + } + state.initState(argumentValues); + calculatedFieldCtx.setState(state); + states.put(ctxId, calculatedFieldCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); + + state.performCalculation(calculatedField.getConfiguration()); + } + + private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { + return switch (calculatedFieldType) { + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 6c2e995b82..1b541bcd5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -51,6 +51,8 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @@ -65,6 +67,8 @@ public class EntityStateSourcingListener { private final TbClusterService tbClusterService; private final TenantService tenantService; + private final CalculatedFieldService calculatedFieldService; + private final DeviceProfileService deviceProfileService; @PostConstruct public void init() { @@ -102,7 +106,7 @@ public class EntityStateSourcingListener { onTenantProfileUpdate(tenantProfile, lifecycleEvent); } case DEVICE -> { - onDeviceUpdate(event.getEntity(), event.getOldEntity()); + onDeviceUpdate(tenantId, event.getEntity(), event.getOldEntity()); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -241,11 +245,19 @@ public class EntityStateSourcingListener { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } - private void onDeviceUpdate(Object entity, Object oldEntity) { + private void onDeviceUpdate(TenantId tenantId, Object entity, Object oldEntity) { Device device = (Device) entity; Device oldDevice = null; if (oldEntity instanceof Device) { oldDevice = (Device) oldEntity; + // TODO: move verification of device type to cluster service + if (!oldDevice.getType().equals(device.getType())) { + DeviceProfile profile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType()); + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profile.getId()); + if (cfExistsByProfile) { + // TODO: send device type updated msg to core + } + } } tbClusterService.onDeviceUpdated(device, oldDevice); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 021a9bbc83..8a5e4cdf65 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -22,16 +22,15 @@ import org.thingsboard.server.common.data.id.EntityId; @Data public class CalculatedFieldCtx { - private CalculatedFieldId calculatedFieldId; - private EntityId entityId; + private CalculatedFieldCtxId id; private CalculatedFieldState state; public CalculatedFieldCtx() { } - public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { - this.calculatedFieldId = calculatedFieldId; - this.entityId = entityId; + public CalculatedFieldCtx(CalculatedFieldCtxId id, CalculatedFieldState state) { + this.id = id; this.state = state; } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java new file mode 100644 index 0000000000..3dc0dead36 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import java.util.UUID; + +public record CalculatedFieldCtxId(UUID cfId, UUID entityId) { +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java new file mode 100644 index 0000000000..adb8b70e7e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +public class CalculatedFieldResult { + + private String name; + private String type; + private AttributeScope scope; + private String value; + + public static CalculatedFieldResult createAttributesResult(String name, AttributeScope scope, String value) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.name = name; + result.type = "ATTRIBUTES"; + result.scope = scope; + result.value = value; + return result; + } + + public static CalculatedFieldResult createTimeSeriesResult(String name, String value) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.name = name; + result.type = "TIME_SERIES"; + result.value = value; + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index 221c44b94c..9997df947f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.entitiy.cf; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -36,6 +37,12 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); + default boolean isValid(Map arguments, CalculatedFieldConfiguration calculatedFieldConfiguration) { + return arguments.keySet().containsAll(calculatedFieldConfiguration.getArguments().keySet()); + } + + void initState(Map argumentValues); + + CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index f2f16de844..0cc644606f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,65 +15,28 @@ */ package org.thingsboard.server.service.entitiy.cf; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.ListeningScheduledExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -84,109 +47,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { private final CalculatedFieldService calculatedFieldService; - private final AttributesService attributesService; - private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; - private ListeningScheduledExecutorService scheduledExecutor; - - private ListeningExecutorService calculatedFieldExecutor; - private ListeningExecutorService calculatedFieldCallbackExecutor; - - private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); - - @Value("${calculatedField.initFetchPackSize:50000}") - @Getter - private int initFetchPackSize; - - @Value("10") - @Getter - private int defaultCalculatedFieldCheckIntervalInSec; - - @PostConstruct - public void init() { - // from AbstractPartitionBasedService - scheduledExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("calculated-field-scheduled"))); - /// - calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); - calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, 0, defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); - } - - @PreDestroy - public void stop() { - // from AbstractPartitionBasedService - if (scheduledExecutor != null) { - scheduledExecutor.shutdown(); - } - /// - if (calculatedFieldExecutor != null) { - calculatedFieldExecutor.shutdownNow(); - } - if (calculatedFieldCallbackExecutor != null) { - calculatedFieldCallbackExecutor.shutdownNow(); - } - } - - @Override - public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - if (proto.getDeleted()) { - log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); - callback.onSuccess(); - } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (proto.getUpdated()) { - log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); - if (!shouldReinit) { - return; - } - } - List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - if (cf != null) { - EntityId entityId = cf.getEntityId(); - calculatedFields.put(calculatedFieldId, cf); - calculatedFieldLinks.put(calculatedFieldId, links); - switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(tenantId, cf, entityId, callback); - } - case ASSET_PROFILE -> { - log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); - PageDataIterable assetIds = new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); - } - case DEVICE_PROFILE -> { - log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); - PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); - } - default -> - throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); - } - } else { - //Calculated field was probably deleted while message was in queue; - log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); - callback.onSuccess(); - } - callback.onSuccess(); - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); - } catch (Exception e) { - log.trace("Failed to process calculated field msg: [{}]", proto, e); - callback.onFailure(e); - } - } @Override public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { @@ -224,58 +84,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - calculatedFieldLinks.remove(calculatedFieldId); - calculatedFields.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); - List statesToRemove = states.keySet().stream() - .filter(key -> key.startsWith(calculatedFieldId.getId().toString())) - .collect(Collectors.toList()); - rocksDBService.deleteAll(statesToRemove); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { - onCalculatedFieldDelete(newCalculatedField.getId(), callback); - } else { - calculatedFields.put(newCalculatedField.getId(), newCalculatedField); - callback.onSuccess(); - shouldReinit = false; - } - return shouldReinit; - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); - - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; - } - - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); - } - private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> @@ -305,70 +113,4 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { - Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { - @Override - public void onSuccess(Optional result) { - String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); - argumentValues.put(key, value); - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to initialize state for entity: [{}]", entityId, t); - callback.onFailure(t); - } - }, calculatedFieldCallbackExecutor)); - - updateOrInitializeState(calculatedField, entityId, argumentValues); - - } - - private ListenableFuture> fetchArgumentValue(TenantId tenantId, BaseCalculatedFieldConfiguration.Argument argument) { - return switch (argument.getType()) { - case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), - MoreExecutors.directExecutor()); - case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), - MoreExecutors.directExecutor()); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - } - - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - String ctxId = generateCtxId(calculatedField.getId(), entityId); - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, - ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); - - CalculatedFieldState state = calculatedFieldCtx.getState(); - if (state != null) { - state.performCalculation(argumentValues, calculatedField.getConfiguration(), false); - } else { - CalculatedFieldState newState = createStateByType(calculatedField.getType()); - newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); - state = newState; - } - calculatedFieldCtx.setState(state); - - states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); - } - - private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { - return switch (calculatedFieldType) { - case SIMPLE -> new SimpleCalculatedFieldState(); - default -> - throw new IllegalArgumentException("Invalid calculated field type '" + calculatedFieldType + "'."); - }; - } - - private String generateCtxId(CalculatedFieldId calculatedFieldId, EntityId entityId) { - return calculatedFieldId.getId() + "_" + entityId.getId(); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java index 2ba3e9bb4f..2cf7aec18d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java @@ -21,6 +21,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.utils.RocksDBConfig; @@ -31,6 +32,7 @@ import java.util.Map; @Service @Slf4j +@ConditionalOnExpression("'${service.type:null}'=='monolith'") public class RocksDBService { private final RocksDB db; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java new file mode 100644 index 0000000000..52435643cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class ScriptCalculatedFieldState implements CalculatedFieldState { + + private TbelInvokeService tbelInvokeService; + + private Map arguments = new HashMap<>(); + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } + + @Override + public void initState(Map argumentValues) { + + } + + @Override + public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java index 8a90893262..c004f9dd65 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.entitiy.cf; import lombok.Data; +import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -26,8 +28,8 @@ import java.util.Map; public class SimpleCalculatedFieldState implements CalculatedFieldState { // TODO: use value object(TsKv) instead of string - Map arguments = new HashMap<>(); - String result; + private Map arguments = new HashMap<>(); + private String outputResult; @Override public CalculatedFieldType getType() { @@ -35,15 +37,30 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation) { - if (initialCalculation) { - // todo: perform initial calculation - this.arguments = argumentValues; - } else { - // todo: perform calculation based on previous data - this.arguments.putAll(argumentValues); + public void initState(Map argumentValues) { + this.arguments = argumentValues; + } + + @Override + public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + if (isValid(arguments, calculatedFieldConfiguration)) { + String expression = calculatedFieldConfiguration.getOutput().getExpression(); + ThreadLocal customExpression = new ThreadLocal<>(); + var expr = customExpression.get(); + if (expr == null) { + expr = new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(arguments.keySet()) + .build(); + customExpression.set(expr); + } + Map variables = new HashMap<>(); + arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + expr.setVariables(variables); + double result = expr.evaluate(); + this.outputResult = Double.toString(result); } - this.result = "result"; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index aa77d29702..89931b8541 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,14 +18,10 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 979c419003..2d3323e097 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -86,7 +86,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; -import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -150,7 +150,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; @@ -179,7 +179,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + ListenableFuture future = deviceActivityEventsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index ebda2cbf82..57d7afcaba 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,6 +21,7 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -136,7 +137,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 4bb67f9f0d..e90f52c5c0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -60,4 +60,6 @@ public interface CalculatedFieldService extends EntityDaoService { boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java new file mode 100644 index 0000000000..bcd22f9216 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +public class Argument { + + private EntityId entityId; + private String key; + private String type; + private String defaultValue; + + private int limit; + private long timeWindow; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 4575e414ac..be69b951d1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -105,14 +105,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel return configNode; } - @Data - public static class Argument { - private EntityId entityId; - private String key; - private String type; - private String defaultValue; - } - @Data public static class Output { private String name; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index f733c35310..a3598cb59d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -39,7 +39,7 @@ public interface CalculatedFieldConfiguration { @JsonIgnore CalculatedFieldType getType(); - Map getArguments(); + Map getArguments(); BaseCalculatedFieldConfiguration.Output getOutput(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index fd669f7865..1ee881ae72 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -195,6 +195,11 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } + @Override + public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldDao.existsByEntityId(tenantId, entityId); + }; + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index d1c9fd86bc..a6b7c2dea1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -34,4 +34,6 @@ public interface CalculatedFieldDao extends Dao { List removeAllByEntityId(TenantId tenantId, EntityId entityId); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index bdc701070d..737a089a15 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -66,6 +66,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao getEntityClass() { return CalculatedFieldEntity.class; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 43b51ce2ac..2082210899 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -880,7 +881,7 @@ public class AssetServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index d5d025a46d..751d883896 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,6 +23,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -149,7 +150,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 1c5e0d8f49..73d398bb39 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,6 +31,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -375,7 +376,7 @@ public class CustomerServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 8f1f16e631..1bd876eae0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -1218,7 +1219,7 @@ public class DeviceServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(device.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); From 40299cab1ac2a54a73c09f342b48c5923d52d99b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 20 Nov 2024 17:34:07 +0200 Subject: [PATCH 031/416] added nofitication on update in telemetry service --- .../cf/CalculatedFieldExecutionService.java | 8 +++ .../cf/CalculatedFieldResult.java | 23 ++---- ...efaultCalculatedFieldExecutionService.java | 70 +++++++++++++++---- .../{entitiy => }/cf/RocksDBService.java | 4 +- .../cf => cf/ctx}/CalculatedFieldCtx.java | 5 +- .../cf => cf/ctx}/CalculatedFieldCtxId.java | 2 +- .../ctx/state}/CalculatedFieldState.java | 14 ++-- .../state}/ScriptCalculatedFieldState.java | 19 +++-- .../state}/SimpleCalculatedFieldState.java | 38 +++++++--- .../cf/DefaultTbCalculatedFieldService.java | 2 +- .../DefaultTelemetrySubscriptionService.java | 60 +++++++++++++++- .../CalculatedFieldControllerTest.java | 9 +-- .../server/dao/cf/CalculatedFieldService.java | 4 ++ .../common/data/cf/CalculatedField.java | 2 + .../cf/CalculatedFieldLinkConfiguration.java | 8 +-- .../data/cf/{ => configuration}/Argument.java | 4 +- .../BaseCalculatedFieldConfiguration.java | 43 ++++++------ .../CalculatedFieldConfiguration.java | 9 ++- .../common/data/cf/configuration/Output.java | 29 ++++++++ .../ScriptCalculatedFieldConfiguration.java | 40 +++++++++++ .../SimpleCalculatedFieldConfiguration.java | 3 +- .../dao/cf/BaseCalculatedFieldService.java | 16 ++++- .../server/dao/cf/CalculatedFieldLinkDao.java | 3 + .../dao/model/sql/CalculatedFieldEntity.java | 7 +- .../sql/cf/CalculatedFieldLinkRepository.java | 2 + ...efaultNativeCalculatedFieldRepository.java | 4 +- .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 6 ++ .../server/dao/service/AssetServiceTest.java | 7 +- .../service/CalculatedFieldServiceTest.java | 9 +-- .../dao/service/CustomerServiceTest.java | 7 +- .../server/dao/service/DeviceServiceTest.java | 7 +- 31 files changed, 349 insertions(+), 115 deletions(-) rename application/src/main/java/org/thingsboard/server/service/{entitiy => }/cf/CalculatedFieldResult.java (53%) rename application/src/main/java/org/thingsboard/server/service/{entitiy => }/cf/RocksDBService.java (98%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx}/CalculatedFieldCtx.java (84%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx}/CalculatedFieldCtxId.java (93%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/CalculatedFieldState.java (70%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/ScriptCalculatedFieldState.java (73%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/SimpleCalculatedFieldState.java (57%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/Argument.java (85%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/BaseCalculatedFieldConfiguration.java (81%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/CalculatedFieldConfiguration.java (81%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/SimpleCalculatedFieldConfiguration.java (90%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6daa88d771..9a8b26d074 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,11 +15,19 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; +import java.util.Map; + public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + +// void onEntityProfileUpdate(TransportProtos.CalculatedFieldEntityProfileUpdateMsgProto proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java rename to application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index adb8b70e7e..4982445735 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -13,34 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import java.util.Map; + @Data public class CalculatedFieldResult { - private String name; private String type; private AttributeScope scope; - private String value; - - public static CalculatedFieldResult createAttributesResult(String name, AttributeScope scope, String value) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.name = name; - result.type = "ATTRIBUTES"; - result.scope = scope; - result.value = value; - return result; - } + private Map resultMap; - public static CalculatedFieldResult createTimeSeriesResult(String name, String value) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.name = name; - result.type = "TIME_SERIES"; - result.value = value; - return result; + public CalculatedFieldResult() { } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c0eeec02bb..250496830c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -29,13 +30,12 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.Argument; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -44,7 +44,10 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.asset.AssetService; @@ -54,12 +57,11 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtx; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtxId; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldState; -import org.thingsboard.server.service.entitiy.cf.RocksDBService; -import org.thingsboard.server.service.entitiy.cf.ScriptCalculatedFieldState; -import org.thingsboard.server.service.entitiy.cf.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; @@ -71,6 +73,9 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.thingsboard.server.common.data.DataConstants.SCOPE; @TbCoreComponent @Service @@ -84,6 +89,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; + private final TbClusterService clusterService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -194,6 +200,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + @Override + public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + try { + CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); + updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); + log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to update time series for calculatedFieldId: [{}]", calculatedFieldId, e); + } + } + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); boolean shouldReinit = true; @@ -250,11 +267,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); + AtomicInteger remaining = new AtomicInteger(arguments.size()); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @Override public void onSuccess(Optional result) { String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); argumentValues.put(key, value); + if (remaining.decrementAndGet() == 0) { + updateOrInitializeState(calculatedField, entityId, argumentValues); + } } @Override @@ -263,15 +284,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas callback.onFailure(t); } }, calculatedFieldCallbackExecutor)); - - updateOrInitializeState(calculatedField, entityId, argumentValues); - } private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { return switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + attributesService.find(tenantId, argument.getEntityId(), argument.getScope(), argument.getKey()), result -> result.map(entry -> (KvEntry) entry), MoreExecutors.directExecutor()); case "TIME_SERIES" -> Futures.transform( @@ -296,7 +314,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - state.performCalculation(calculatedField.getConfiguration()); + CalculatedFieldResult result = state.performCalculation(calculatedField.getConfiguration()); + if (result != null) { + pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + } + } + + private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { + try { + String type = calculatedFieldResult.getType(); + TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + ObjectNode jsonNodes = createJsonPayload(calculatedFieldResult); + TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(jsonNodes)); + clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); + } catch (Exception e) { + log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); + } + } + + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { + ObjectNode jsonNodes = JacksonUtil.newObjectNode(); + calculatedFieldResult.getResultMap().forEach(jsonNodes::put); + return jsonNodes; } private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java similarity index 98% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java rename to application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index 2cf7aec18d..d6b2980042 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf; import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDB; @@ -94,4 +94,4 @@ public class RocksDBService { return map; } -} \ No newline at end of file +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java index 8a5e4cdf65..4b2a6c918f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx; import lombok.Data; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data public class CalculatedFieldCtx { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java index 3dc0dead36..a316c54b76 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java similarity index 70% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 9997df947f..c25a6960ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -13,14 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -30,15 +31,16 @@ import java.util.Map; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE") + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT") }) public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map arguments, CalculatedFieldConfiguration calculatedFieldConfiguration) { - return arguments.keySet().containsAll(calculatedFieldConfiguration.getArguments().keySet()); + default boolean isValid(Map argumentValues, Map arguments) { + return argumentValues.keySet().containsAll(arguments.keySet()); } void initState(Map argumentValues); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 52435643cc..238e8005f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -13,23 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; -import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; import java.util.Map; @Data +@Slf4j public class ScriptCalculatedFieldState implements CalculatedFieldState { - private TbelInvokeService tbelInvokeService; - private Map arguments = new HashMap<>(); + public ScriptCalculatedFieldState() { + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -37,11 +40,15 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @Override public void initState(Map argumentValues) { - + if (arguments == null) { + this.arguments = new HashMap<>(); + } + this.arguments.putAll(argumentValues); } @Override public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + // TODO: implement return null; } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index c004f9dd65..e984e300c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -13,13 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; import java.util.Map; @@ -28,8 +31,7 @@ import java.util.Map; public class SimpleCalculatedFieldState implements CalculatedFieldState { // TODO: use value object(TsKv) instead of string - private Map arguments = new HashMap<>(); - private String outputResult; + private Map arguments; @Override public CalculatedFieldType getType() { @@ -38,29 +40,43 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { @Override public void initState(Map argumentValues) { - this.arguments = argumentValues; + if (arguments == null) { + arguments = new HashMap<>(); + } + arguments.putAll(argumentValues); } @Override public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { - if (isValid(arguments, calculatedFieldConfiguration)) { - String expression = calculatedFieldConfiguration.getOutput().getExpression(); + Output output = calculatedFieldConfiguration.getOutput(); + Map arguments = calculatedFieldConfiguration.getArguments(); + + if (isValid(this.arguments, arguments)) { + CalculatedFieldResult result = new CalculatedFieldResult(); + String expression = output.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { expr = new ExpressionBuilder(expression) .implicitMultiplication(true) - .variables(arguments.keySet()) + .variables(this.arguments.keySet()) .build(); customExpression.set(expr); } Map variables = new HashMap<>(); - arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); expr.setVariables(variables); - double result = expr.evaluate(); - this.outputResult = Double.toString(result); + + String expressionResult = String.valueOf(expr.evaluate()); + + result.setType(output.getType()); + result.setScope(output.getScope()); + result.setResultMap(Map.of(output.getName(), expressionResult)); + return result; } + return null; + // TODO: handle what happens when not valid } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 0cc644606f..4d28ff55ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 5513c41929..ac769bee9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -32,6 +32,8 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -40,6 +42,7 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -47,9 +50,11 @@ import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -77,6 +82,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; + private final CalculatedFieldService calculatedFieldService; + private final CalculatedFieldExecutionService calculatedFieldExecutionService; private ExecutorService tsCallBackExecutor; @@ -87,12 +94,16 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TimeseriesService tsService, @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, - TbApiUsageStateService apiUsageStateService) { + TbApiUsageStateService apiUsageStateService, + CalculatedFieldService calculatedFieldService, + CalculatedFieldExecutionService calculatedFieldExecutionService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; + this.calculatedFieldService = calculatedFieldService; + this.calculatedFieldExecutionService = calculatedFieldExecutionService; } @PostConstruct @@ -179,6 +190,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addEntityViewCallback(tenantId, entityId, ts); + updateTelemetryInCalculatedFields(tenantId, entityId, ts); } private void saveWithoutLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { @@ -187,6 +199,49 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } + private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { + if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { + List cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + if (!cfLinks.isEmpty()) { + cfLinks.forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + Map attributes = link.getConfiguration().getAttributes(); + Map timeSeries = link.getConfiguration().getTimeSeries(); + List filteredTelemetry = telemetry.stream() + .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .toList(); + + + Map updatedTelemetry = new HashMap<>(); + for (KvEntry telemetryEntry : filteredTelemetry) { + String key = telemetryEntry.getKey(); + if (telemetryEntry instanceof AttributeKvEntry) { + for (Map.Entry attribute : attributes.entrySet()) { + if (telemetryEntry.getKey().equals(attribute.getValue())) { + key = attribute.getKey(); + break; + } + } + } + if (telemetryEntry instanceof TsKvEntry) { + for (Map.Entry timeSeriesEntry : timeSeries.entrySet()) { + if (telemetryEntry.getKey().equals(timeSeriesEntry.getValue())) { + key = timeSeriesEntry.getKey(); + break; + } + } + } + updatedTelemetry.put(key, telemetryEntry.getValueAsString()); + } + + if (!updatedTelemetry.isEmpty()) { + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); + } + }); + } + } + } + private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), @@ -263,6 +318,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); + updateTelemetryInCalculatedFields(tenantId, entityId, attributes); } @Override @@ -270,6 +326,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope.name(), attributes, notifyDevice)); + updateTelemetryInCalculatedFields(tenantId, entityId, attributes); } @Override @@ -283,6 +340,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); + updateTelemetryInCalculatedFields(tenantId, entityId, ts); } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 57d7afcaba..314dc2bdba 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,11 +21,12 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; @@ -144,7 +145,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index e90f52c5c0..e44ff0ba22 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -54,12 +54,16 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + boolean referencedInAnyCalculatedFieldIncludingEntityId(TenantId tenantId, EntityId referencedEntityId); + boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index ceb1222fe2..e626c9d3d2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index 02d668a67a..c5f81cd572 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -17,13 +17,13 @@ package org.thingsboard.server.common.data.cf; import lombok.Data; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; @Data public class CalculatedFieldLinkConfiguration { - private List attributes = new ArrayList<>(); - private List timeSeries = new ArrayList<>(); + private Map attributes = new HashMap<>(); + private Map timeSeries = new HashMap<>(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java similarity index 85% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index bcd22f9216..f34f5e9cb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.EntityId; @Data @@ -24,6 +25,7 @@ public class Argument { private EntityId entityId; private String key; private String type; + private AttributeScope scope; private String defaultValue; private int limit; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java similarity index 81% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index be69b951d1..7692d792f8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -13,14 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -60,18 +62,20 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel @Override public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); - arguments.values().stream() - .filter(argument -> argument.getEntityId().equals(entityId)) - .forEach(argument -> { - switch (argument.getType()) { - case "ATTRIBUTES": - linkConfiguration.getAttributes().add(argument.getKey()); - break; - case "TIME_SERIES": - linkConfiguration.getTimeSeries().add(argument.getKey()); - break; - } - }); + + for (Map.Entry entry : arguments.entrySet()) { + Argument argument = entry.getValue(); + if (argument.getEntityId().equals(entityId)) { + switch (argument.getType()) { + case "ATTRIBUTES": + linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); + break; + case "TIME_SERIES": + linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); + break; + } + } + } return linkConfiguration; } @@ -93,25 +97,21 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argumentNode.put("key", argument.getKey()); argumentNode.put("type", argument.getType()); + argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); }); if (output != null) { ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("name", output.getName()); outputNode.put("type", output.getType()); + outputNode.put("scope", String.valueOf(output.getScope())); outputNode.put("expression", output.getExpression()); } return configNode; } - @Data - public static class Output { - private String name; - private String type; - private String expression; - } - private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { if (config == null || !config.isObject()) { return null; @@ -133,6 +133,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argument.setKey(argumentNode.get("key").asText()); argument.setType(argumentNode.get("type").asText()); + argument.setScope(AttributeScope.valueOf(argumentNode.get("scope").asText())); argument.setDefaultValue(argumentNode.get("defaultValue").asText()); arguments.put(key, argument); }); @@ -142,7 +143,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel JsonNode outputNode = config.get("output"); if (outputNode != null) { Output output = new Output(); + output.setName(outputNode.get("name").asText()); output.setType(outputNode.get("type").asText()); + output.setScope(AttributeScope.valueOf(outputNode.get("scope").asText())); output.setExpression(outputNode.get("expression").asText()); this.setOutput(output); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java similarity index 81% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index a3598cb59d..155015028f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -13,13 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; import java.util.List; @@ -32,7 +34,8 @@ import java.util.UUID; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE") + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") }) public interface CalculatedFieldConfiguration { @@ -41,7 +44,7 @@ public interface CalculatedFieldConfiguration { Map getArguments(); - BaseCalculatedFieldConfiguration.Output getOutput(); + Output getOutput(); @JsonIgnore List getReferencedEntities(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java new file mode 100644 index 0000000000..683e372ebc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +public class Output { + + private String name; + private String type; + private AttributeScope scope; + private String expression; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..a24328b4c9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.UUID; + +@Data +public class ScriptCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public ScriptCalculatedFieldConfiguration() { + super(); + } + + public ScriptCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java similarity index 90% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java index 327f9cdc75..af11d2f5d8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 1ee881ae72..cf5f79c5fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -21,7 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -173,6 +173,12 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); } + @Override + public List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findAllCalculatedFieldLinksByEntityId, entityId [{}]", entityId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByEntityId(tenantId, entityId); + } + @Override public ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing findAllCalculatedFieldLinksByIdAsync, calculatedFieldId [{}]", calculatedFieldId); @@ -195,6 +201,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } + @Override + public boolean referencedInAnyCalculatedFieldIncludingEntityId(TenantId tenantId, EntityId referencedEntityId) { + return calculatedFieldDao.findAllByTenantId(tenantId).stream() + .map(CalculatedField::getConfiguration) + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); + } + @Override public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByEntityId(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 34f2129bd7..549db510ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.cf; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -29,6 +30,8 @@ public interface CalculatedFieldLinkDao extends Dao { List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); List findAll(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 3c45a81cb5..6500d2a1e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,9 +24,10 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -122,6 +123,8 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem switch (CalculatedFieldType.valueOf(type)) { case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT: + return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index 61c4026cca..d7325df8d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -25,4 +25,6 @@ public interface CalculatedFieldLinkRepository extends JpaRepository findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 417a468b2c..eebca14b6e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -25,11 +25,11 @@ import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index 417b529dc9..29492a10cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +50,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + @Override public ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { return service.submit(() -> findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 2082210899..2012090264 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,10 +30,11 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -888,7 +889,7 @@ public class AssetServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 751d883896..2bdb1b8897 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,11 +23,12 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.cf.CalculatedFieldService; @@ -157,7 +158,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 73d398bb39..b58f739462 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,10 +31,11 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -383,7 +384,7 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 1bd876eae0..38bd21170a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,10 +39,11 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -1226,7 +1227,7 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); From c75603f57c0d093d14cb7c3cf4fe187082f70ca5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 21 Nov 2024 15:45:39 +0200 Subject: [PATCH 032/416] added implementation for script type --- .../cf/CalculatedFieldExecutionService.java | 5 +- .../service/cf/CalculatedFieldResult.java | 2 +- ...efaultCalculatedFieldExecutionService.java | 43 ++++++--- .../state/CalculatedFieldScriptEngine.java | 29 ++++++ .../cf/ctx/state/CalculatedFieldState.java | 12 ++- .../CalculatedFieldTbelScriptEngine.java | 90 +++++++++++++++++++ .../ctx/state/ScriptCalculatedFieldState.java | 59 ++++++++++-- .../ctx/state/SimpleCalculatedFieldState.java | 22 ++--- .../DefaultTelemetrySubscriptionService.java | 47 +++++----- .../thingsboard/script/api/ScriptType.java | 2 +- 10 files changed, 248 insertions(+), 63 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 9a8b26d074..d4e0d1da84 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; @@ -26,8 +27,6 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); - -// void onEntityProfileUpdate(TransportProtos.CalculatedFieldEntityProfileUpdateMsgProto proto, TbCallback callback); + void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 4982445735..87f1d08a84 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -25,7 +25,7 @@ public class CalculatedFieldResult { private String type; private AttributeScope scope; - private Map resultMap; + private Map resultMap; public CalculatedFieldResult() { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 250496830c..c83bd7a71a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; @@ -30,6 +31,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -90,6 +92,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; private final TbClusterService clusterService; + private final TbelInvokeService tbelInvokeService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -201,7 +204,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); @@ -266,13 +269,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); + Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @Override public void onSuccess(Optional result) { - String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); - argumentValues.put(key, value); + // todo: should be rewritten implementation for default value + argumentValues.put(key, result.orElse(null)); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -300,7 +303,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -314,10 +317,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - CalculatedFieldResult result = state.performCalculation(calculatedField.getConfiguration()); - if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); - } + ListenableFuture resultFuture = state.performCalculation(calculatedField.getTenantId(), calculatedField.getConfiguration(), tbelInvokeService); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedField.getId(), entityId, t); + } + }, MoreExecutors.directExecutor()); + } private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { @@ -325,8 +339,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas String type = calculatedFieldResult.getType(); TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; - ObjectNode jsonNodes = createJsonPayload(calculatedFieldResult); - TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(jsonNodes)); + ObjectNode payload = createJsonPayload(calculatedFieldResult); + TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); @@ -334,9 +348,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { - ObjectNode jsonNodes = JacksonUtil.newObjectNode(); - calculatedFieldResult.getResultMap().forEach(jsonNodes::put); - return jsonNodes; + ObjectNode payload = JacksonUtil.newObjectNode(); + Map resultMap = calculatedFieldResult.getResultMap(); + resultMap.forEach((k, v) -> payload.set(k, JacksonUtil.convertValue(v, JsonNode.class))); + return payload; } private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java new file mode 100644 index 0000000000..6b54536019 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.Map; + +public interface CalculatedFieldScriptEngine { + + ListenableFuture executeScriptAsync(Map arguments); + + void destroy(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index c25a6960ac..dffcf09820 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -18,9 +18,13 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -39,12 +43,12 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map argumentValues, Map arguments) { + default boolean isValid(Map argumentValues, Map arguments) { return argumentValues.keySet().containsAll(arguments.keySet()); } - void initState(Map argumentValues); + void initState(Map argumentValues); - CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration); + ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java new file mode 100644 index 0000000000..7e8376be8e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.ScriptType; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import javax.script.ScriptException; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +@Slf4j +public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEngine { + + private final TbelInvokeService tbelInvokeService; + + private final UUID scriptId; + private final TenantId tenantId; + + public CalculatedFieldTbelScriptEngine(TenantId tenantId, TbelInvokeService tbelInvokeService, String script, String... argNames) { + this.tenantId = tenantId; + this.tbelInvokeService = tbelInvokeService; + try { + this.scriptId = this.tbelInvokeService.eval(tenantId, ScriptType.CALCULATED_FIELD_SCRIPT, script, argNames).get(); + } catch (Exception e) { + Throwable t = e; + if (e instanceof ExecutionException) { + t = e.getCause(); + } + throw new IllegalArgumentException("Can't compile script: " + t.getMessage(), t); + } + } + + @Override + public ListenableFuture executeScriptAsync(Map arguments) { + log.trace("execute script async, arguments {}", arguments); + Object[] args = new Object[arguments.size()]; + int index = 0; + for (KvEntry entry : arguments.values()) { + switch (entry.getDataType()) { + case BOOLEAN -> args[index] = entry.getBooleanValue().orElse(null); + case DOUBLE -> args[index] = entry.getDoubleValue().orElse(null); + case LONG -> args[index] = entry.getLongValue().orElse(null); + case JSON -> args[index] = entry.getJsonValue().map(JacksonUtil::toJsonNode).orElse(null); + default -> args[index] = entry.getValueAsString(); + } + index++; + } + return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), + o -> { + try { + return Futures.immediateFuture(o); + } catch (Exception e) { + if (e.getCause() instanceof ScriptException) { + return Futures.immediateFailedFuture(e.getCause()); + } else if (e.getCause() instanceof RuntimeException) { + return Futures.immediateFailedFuture(new ScriptException(e.getCause().getMessage())); + } else { + return Futures.immediateFailedFuture(new ScriptException(e)); + } + } + }, MoreExecutors.directExecutor()); + } + + @Override + public void destroy() { + tbelInvokeService.release(this.scriptId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 238e8005f2..363c237df5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -15,10 +15,19 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -28,7 +37,10 @@ import java.util.Map; @Slf4j public class ScriptCalculatedFieldState implements CalculatedFieldState { - private Map arguments = new HashMap<>(); + @JsonIgnore + private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + + private Map arguments = new HashMap<>(); public ScriptCalculatedFieldState() { } @@ -39,7 +51,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { this.arguments = new HashMap<>(); } @@ -47,9 +59,46 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { } @Override - public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { - // TODO: implement - return null; + public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + if (calculatedFieldScriptEngine == null) { + initEngine(tenantId, calculatedFieldConfiguration, tbelInvokeService); + } + + ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); + + return Futures.transform(resultFuture, result -> { + Output output = calculatedFieldConfiguration.getOutput(); + Map resultMap = new HashMap<>(); + + if (result instanceof Map) { + Map map = JacksonUtil.convertValue(result, Map.class); + if (map != null) { + resultMap.putAll(map); + } + } else { + resultMap.put(output.getName(), JacksonUtil.convertValue(result, Object.class)); + } + + CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); + calculatedFieldResult.setType(output.getType()); + calculatedFieldResult.setScope(output.getScope()); + calculatedFieldResult.setResultMap(resultMap); + + return calculatedFieldResult; + }, MoreExecutors.directExecutor()); + } + + private void initEngine(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( + tenantId, + tbelInvokeService, + calculatedFieldConfiguration.getOutput().getExpression(), + arguments.keySet().toArray(new String[0]) + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index e984e300c6..725da6c7a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -15,13 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -30,8 +35,7 @@ import java.util.Map; @Data public class SimpleCalculatedFieldState implements CalculatedFieldState { - // TODO: use value object(TsKv) instead of string - private Map arguments; + private Map arguments; @Override public CalculatedFieldType getType() { @@ -39,7 +43,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } @@ -47,7 +51,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { Output output = calculatedFieldConfiguration.getOutput(); Map arguments = calculatedFieldConfiguration.getArguments(); @@ -64,19 +68,17 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValueAsString()))); expr.setVariables(variables); - String expressionResult = String.valueOf(expr.evaluate()); + double expressionResult = expr.evaluate(); result.setType(output.getType()); result.setScope(output.getScope()); result.setResultMap(Map.of(output.getName(), expressionResult)); - return result; + return Futures.immediateFuture(result); } - return null; - // TODO: handle what happens when not valid } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ac769bee9e..76c1383f6a 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -69,6 +69,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -207,32 +208,28 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); Map timeSeries = link.getConfiguration().getTimeSeries(); - List filteredTelemetry = telemetry.stream() + Map updatedTelemetry = telemetry.stream() .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) - .toList(); - - - Map updatedTelemetry = new HashMap<>(); - for (KvEntry telemetryEntry : filteredTelemetry) { - String key = telemetryEntry.getKey(); - if (telemetryEntry instanceof AttributeKvEntry) { - for (Map.Entry attribute : attributes.entrySet()) { - if (telemetryEntry.getKey().equals(attribute.getValue())) { - key = attribute.getKey(); - break; - } - } - } - if (telemetryEntry instanceof TsKvEntry) { - for (Map.Entry timeSeriesEntry : timeSeries.entrySet()) { - if (telemetryEntry.getKey().equals(timeSeriesEntry.getValue())) { - key = timeSeriesEntry.getKey(); - break; - } - } - } - updatedTelemetry.put(key, telemetryEntry.getValueAsString()); - } + .collect(Collectors.toMap( + entry -> { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + }, + entry -> entry, + (v1, v2) -> v1 + )); if (!updatedTelemetry.isEmpty()) { calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java index cdcdf815d0..7f8c513957 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java @@ -16,5 +16,5 @@ package org.thingsboard.script.api; public enum ScriptType { - RULE_NODE_SCRIPT + RULE_NODE_SCRIPT, CALCULATED_FIELD_SCRIPT } From 6ed4abb9038eb424c129965d99fae7ed828c633a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 21 Nov 2024 17:39:16 +0200 Subject: [PATCH 033/416] separated expression from output --- ...efaultCalculatedFieldExecutionService.java | 4 +-- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- .../CalculatedFieldControllerTest.java | 8 +++--- .../BaseCalculatedFieldConfiguration.java | 27 +++++++++++++++---- .../CalculatedFieldConfiguration.java | 2 ++ .../common/data/cf/configuration/Output.java | 1 - ...efaultNativeCalculatedFieldRepository.java | 5 +++- .../server/dao/service/AssetServiceTest.java | 6 +++-- .../service/CalculatedFieldServiceTest.java | 8 +++--- .../dao/service/CustomerServiceTest.java | 6 +++-- .../server/dao/service/DeviceServiceTest.java | 6 +++-- 12 files changed, 54 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c83bd7a71a..6aa9997d1f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -253,9 +253,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); + boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } private void fetchCalculatedFields() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 363c237df5..cbee8dc902 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -96,7 +96,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( tenantId, tbelInvokeService, - calculatedFieldConfiguration.getOutput().getExpression(), + calculatedFieldConfiguration.getExpression(), arguments.keySet().toArray(new String[0]) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 725da6c7a7..08c4741de5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -57,7 +57,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { if (isValid(this.arguments, arguments)) { CalculatedFieldResult result = new CalculatedFieldResult(); - String expression = output.getExpression(); + String expression = calculatedFieldConfiguration.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 314dc2bdba..77ca268d12 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,10 +21,10 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; @@ -145,9 +145,11 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 7692d792f8..55a43a00ce 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -40,6 +40,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel private final ObjectMapper mapper = new ObjectMapper(); protected Map arguments; + protected String expression; protected Output output; public BaseCalculatedFieldConfiguration() { @@ -48,6 +49,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); this.arguments = calculatedFieldConfig.getArguments(); + this.expression = calculatedFieldConfig.getExpression(); this.output = calculatedFieldConfig.getOutput(); } @@ -101,12 +103,17 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("defaultValue", argument.getDefaultValue()); }); + if (expression != null) { + configNode.put("expression", expression); + } + if (output != null) { ObjectNode outputNode = configNode.putObject("output"); outputNode.put("name", output.getName()); outputNode.put("type", output.getType()); - outputNode.put("scope", String.valueOf(output.getScope())); - outputNode.put("expression", output.getExpression()); + if (output.getScope() != null) { + outputNode.put("scope", String.valueOf(output.getScope())); + } } return configNode; @@ -133,20 +140,30 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argument.setKey(argumentNode.get("key").asText()); argument.setType(argumentNode.get("type").asText()); - argument.setScope(AttributeScope.valueOf(argumentNode.get("scope").asText())); + JsonNode scope = argumentNode.get("scope"); + if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { + argument.setScope(AttributeScope.valueOf(scope.asText())); + } argument.setDefaultValue(argumentNode.get("defaultValue").asText()); arguments.put(key, argument); }); } this.setArguments(arguments); + JsonNode expressionNode = config.get("expression"); + if (expressionNode != null && expressionNode.isTextual()) { + this.setExpression(expressionNode.asText()); + } + JsonNode outputNode = config.get("output"); if (outputNode != null) { Output output = new Output(); output.setName(outputNode.get("name").asText()); output.setType(outputNode.get("type").asText()); - output.setScope(AttributeScope.valueOf(outputNode.get("scope").asText())); - output.setExpression(outputNode.get("expression").asText()); + JsonNode scope = outputNode.get("scope"); + if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { + output.setScope(AttributeScope.valueOf(scope.asText())); + } this.setOutput(output); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 155015028f..5c428bd628 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -44,6 +44,8 @@ public interface CalculatedFieldConfiguration { Map getArguments(); + String getExpression(); + Output getOutput(); @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 683e372ebc..46257d1ccc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -24,6 +24,5 @@ public class Output { private String name; private String type; private AttributeScope scope; - private String expression; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index eebca14b6e..fc40f72c93 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -25,10 +25,11 @@ import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -138,6 +139,8 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF switch (type) { case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT: + return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 2012090264..87f2cb1f45 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,9 +30,9 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; @@ -889,9 +889,11 @@ public class AssetServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 2bdb1b8897..77ed026b1d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,10 +23,10 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -158,9 +158,11 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index b58f739462..94c8440057 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,9 +31,9 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; @@ -384,9 +384,11 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 38bd21170a..d5394a3494 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,9 +39,9 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; @@ -1227,9 +1227,11 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); From 31103e90e3e3cb5f440823872f2a0c85cdac6b10 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 22 Nov 2024 17:17:59 +0200 Subject: [PATCH 034/416] added implementation to handle update entity profiles events --- .../cf/CalculatedFieldExecutionService.java | 2 + ...efaultCalculatedFieldExecutionService.java | 74 ++++++++++++++++--- .../CalculatedFieldTbelScriptEngine.java | 14 +--- .../ctx/state/ScriptCalculatedFieldState.java | 13 +--- .../entitiy/EntityStateSourcingListener.java | 31 ++++---- .../queue/DefaultTbClusterService.java | 44 ++++++++++- .../queue/DefaultTbCoreConsumerService.java | 19 ++++- .../server/cluster/TbClusterService.java | 3 + .../server/dao/cf/CalculatedFieldService.java | 2 + common/proto/src/main/proto/queue.proto | 34 ++++++--- .../server/dao/asset/BaseAssetService.java | 4 +- .../dao/cf/BaseCalculatedFieldService.java | 14 +++- .../server/dao/cf/CalculatedFieldDao.java | 3 + .../dao/sql/cf/CalculatedFieldRepository.java | 3 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 6 ++ 15 files changed, 202 insertions(+), 64 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index d4e0d1da84..6b7b12655f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,4 +29,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 6aa9997d1f..9edea50a05 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -27,12 +27,14 @@ import jakarta.annotation.PreDestroy; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -44,8 +46,14 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -210,7 +218,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { - log.trace("Failed to update time series for calculatedFieldId: [{}]", calculatedFieldId, e); + log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); + } + } + + @Override + public void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); + EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); + + log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + + List cfIdsOfOldProfile = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId); + cfIdsOfOldProfile.forEach(id -> states.remove(new CalculatedFieldCtxId(id.getId(), entityId.getId()))); + List ctxIdsToDelete = cfIdsOfOldProfile.stream().map(cfId -> JacksonUtil.writeValueAsString(new CalculatedFieldCtxId(cfId.getId(), entityId.getId()))).toList(); + rocksDBService.deleteAll(ctxIdsToDelete); + + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) + .forEach(cfId -> { + CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + }); + + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) + .stream() + .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) + .forEach(cf -> initializeStateForEntity(tenantId, cf, entityId, callback)); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); } } @@ -271,10 +310,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument, entityId), new FutureCallback<>() { @Override public void onSuccess(Optional result) { - // todo: should be rewritten implementation for default value argumentValues.put(key, result.orElse(null)); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); @@ -289,20 +327,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { + private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument, EntityId targetEntityId) { + EntityId argumentEntityId = argument.getEntityId(); + EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; return switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), argument.getScope(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of( + new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) + )), MoreExecutors.directExecutor()); case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of( + new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) + )), MoreExecutors.directExecutor()); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; } + private KvEntry createDefaultKvEntry(Argument argument) { + String key = argument.getKey(); + String defaultValue = argument.getDefaultValue(); + if (NumberUtils.isParsable(defaultValue)) { + return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); + } + if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { + return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); + } + return new StringDataEntry(key, defaultValue); + } + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -322,7 +378,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + pushMsgToRuleEngine(calculatedField.getTenantId(), entityId, result); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 7e8376be8e..5cc58b2a95 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,7 +19,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -55,18 +54,7 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng @Override public ListenableFuture executeScriptAsync(Map arguments) { log.trace("execute script async, arguments {}", arguments); - Object[] args = new Object[arguments.size()]; - int index = 0; - for (KvEntry entry : arguments.values()) { - switch (entry.getDataType()) { - case BOOLEAN -> args[index] = entry.getBooleanValue().orElse(null); - case DOUBLE -> args[index] = entry.getDoubleValue().orElse(null); - case LONG -> args[index] = entry.getLongValue().orElse(null); - case JSON -> args[index] = entry.getJsonValue().map(JacksonUtil::toJsonNode).orElse(null); - default -> args[index] = entry.getValueAsString(); - } - index++; - } + Object[] args = arguments.values().stream().map(KvEntry::getValue).toArray(); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index cbee8dc902..5c984a8d16 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -72,16 +72,9 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { return Futures.transform(resultFuture, result -> { Output output = calculatedFieldConfiguration.getOutput(); - Map resultMap = new HashMap<>(); - - if (result instanceof Map) { - Map map = JacksonUtil.convertValue(result, Map.class); - if (map != null) { - resultMap.putAll(map); - } - } else { - resultMap.put(output.getName(), JacksonUtil.convertValue(result, Object.class)); - } + Map resultMap = result instanceof Map + ? JacksonUtil.convertValue(result, Map.class) + : new HashMap<>(); CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); calculatedFieldResult.setType(output.getType()); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 1b541bcd5d..154f5f4833 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; @@ -51,8 +52,6 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; -import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @@ -67,8 +66,6 @@ public class EntityStateSourcingListener { private final TbClusterService tbClusterService; private final TenantService tenantService; - private final CalculatedFieldService calculatedFieldService; - private final DeviceProfileService deviceProfileService; @PostConstruct public void init() { @@ -88,7 +85,10 @@ public class EntityStateSourcingListener { ComponentLifecycleEvent lifecycleEvent = isCreated ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED; switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { + case ASSET -> { + onAssetUpdate(event.getEntity(), event.getOldEntity()); + } + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -106,7 +106,7 @@ public class EntityStateSourcingListener { onTenantProfileUpdate(tenantProfile, lifecycleEvent); } case DEVICE -> { - onDeviceUpdate(tenantId, event.getEntity(), event.getOldEntity()); + onDeviceUpdate(event.getEntity(), event.getOldEntity()); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -245,23 +245,24 @@ public class EntityStateSourcingListener { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } - private void onDeviceUpdate(TenantId tenantId, Object entity, Object oldEntity) { + private void onDeviceUpdate(Object entity, Object oldEntity) { Device device = (Device) entity; Device oldDevice = null; if (oldEntity instanceof Device) { oldDevice = (Device) oldEntity; - // TODO: move verification of device type to cluster service - if (!oldDevice.getType().equals(device.getType())) { - DeviceProfile profile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType()); - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profile.getId()); - if (cfExistsByProfile) { - // TODO: send device type updated msg to core - } - } } tbClusterService.onDeviceUpdated(device, oldDevice); } + private void onAssetUpdate(Object entity, Object oldEntity) { + Asset asset = (Asset) entity; + Asset oldAsset = null; + if (oldEntity instanceof Asset) { + oldAsset = (Asset) oldEntity; + } + tbClusterService.onAssetUpdated(asset, oldAsset); + } + private void onEdgeEvent(TenantId tenantId, EntityId entityId, Object entity, ComponentLifecycleEvent lifecycleEvent) { if (entity instanceof Edge) { tbClusterService.onEdgeStateChangeEvent(new ComponentLifecycleMsg(tenantId, entityId, lifecycleEvent)); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f7c8230716..58a16d9c0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -68,6 +68,7 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -149,6 +150,7 @@ public class DefaultTbClusterService implements TbClusterService { private final GatewayNotificationsService gatewayNotificationsService; private final EdgeService edgeService; private final TbTransactionalCache edgeIdServiceIdCache; + private final CalculatedFieldService calculatedFieldService; @Override public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { @@ -609,7 +611,11 @@ public class DefaultTbClusterService implements TbClusterService { if (deviceNameChanged) { gatewayNotificationsService.onDeviceUpdated(device, old); } - if (deviceNameChanged || !device.getType().equals(old.getType())) { + boolean deviceTypeChanged = !device.getType().equals(old.getType()); + if (deviceTypeChanged) { + handleProfileChange(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + } + if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } @@ -618,6 +624,26 @@ public class DefaultTbClusterService implements TbClusterService { otaPackageStateService.update(device, old); } + @Override + public void onAssetUpdated(Asset asset, Asset old) { + var created = old == null; + broadcastEntityChangeToTransport(asset.getTenantId(), asset.getId(), asset, null); + if (old != null) { + boolean assetTypeChanged = !asset.getType().equals(old.getType()); + if (assetTypeChanged) { + handleProfileChange(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + } + } + broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + private void handleProfileChange(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + if (cfExistsByProfile) { + sendEntityTypeUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + } + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { @@ -775,4 +801,20 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } + private void sendEntityTypeUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setEntityProfileType(newProfileId.getEntityType().name()); + builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); + builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); + builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); + TransportProtos.EntityProfileUpdateMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2d3323e097..3caaab6613 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -157,6 +158,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> firmwareStatesConsumer; private volatile ListeningExecutorService deviceActivityEventsExecutor; + private volatile ListeningExecutorService calculatedFieldsExecutor; public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, @@ -202,6 +204,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) @@ -315,6 +318,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { @@ -677,6 +682,18 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityTypeChanged(profileUpdateMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process device type updated message for device [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index f173005107..131b50a52c 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -97,6 +98,8 @@ public interface TbClusterService extends TbQueueClusterService { void onDeviceAssignedToTenant(TenantId oldTenantId, Device device); + void onAssetUpdated(Asset asset, Asset old); + void onResourceChange(TbResourceInfo resource, TbQueueCallback callback); void onResourceDeleted(TbResourceInfo resource, TbQueueCallback callback); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index e44ff0ba22..1e64fdac60 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -36,6 +36,8 @@ public interface CalculatedFieldService extends EntityDaoService { ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 95e4fa8601..bf49a68c80 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -771,6 +771,29 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } +message CalculatedFieldMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} + +message EntityProfileUpdateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + string entityProfileType = 6; + int64 oldProfileIdMSB = 7; + int64 oldProfileIdLSB = 8; + int64 newProfileIdMSB = 9; + int64 newProfileIdLSB = 10; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1267,16 +1290,6 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } -message CalculatedFieldMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} - /** TB Core to Version Control Service */ @@ -1513,6 +1526,7 @@ message ToCoreMsg { DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; + EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 2135792174..7242cfde68 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -177,8 +177,8 @@ public class BaseAssetService extends AbstractCachedEntityService findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldIdsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldIdsByEntityId(tenantId, entityId); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); @@ -133,7 +141,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements public int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { log.trace("Executing deleteAllCalculatedFieldsByEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); - validateId(entityId.getId(), id -> "Incorrect entityId " + id); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); List calculatedFields = calculatedFieldDao.removeAllByEntityId(tenantId, entityId); return calculatedFields.size(); } @@ -212,7 +220,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByEntityId(tenantId, entityId); - }; + } @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index a6b7c2dea1..5b3bcc2750 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -28,6 +29,8 @@ public interface CalculatedFieldDao extends Dao { List findAllByTenantId(TenantId tenantId); + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 333057e8c5..9aa0aee428 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import java.util.List; @@ -25,6 +26,8 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 737a089a15..e3762f6157 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -49,6 +50,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.findCalculatedFieldIdsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); From 2c7c6f0c5edfe140a1ce0e528824ec58c48b5647 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 25 Nov 2024 08:57:09 +0200 Subject: [PATCH 035/416] fixed tests for cluster service --- .../server/service/queue/DefaultTbClusterServiceTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java index 25fe589a08..cb61980cf4 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; @@ -102,6 +103,8 @@ public class DefaultTbClusterServiceTest { protected TbRuleEngineProducerService ruleEngineProducerService; @MockBean protected TbTransactionalCache edgeCache; + @MockBean + protected CalculatedFieldService calculatedFieldService; @SpyBean protected TopicService topicService; From c6d91c4ce870886e4af404246020dba15e089dd7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 25 Nov 2024 17:28:44 +0200 Subject: [PATCH 036/416] added last records type of cf --- ...efaultCalculatedFieldExecutionService.java | 84 ++++++++++++++--- .../service/cf/ctx/state/ArgumentEntry.java | 30 ++++++ .../cf/ctx/state/CalculatedFieldState.java | 10 +- .../cf/ctx/state/CalculationContext.java | 35 +++++++ .../LastRecordsCalculatedFieldState.java | 91 +++++++++++++++++++ .../ctx/state/ScriptCalculatedFieldState.java | 17 ++-- .../ctx/state/SimpleCalculatedFieldState.java | 13 ++- .../common/data/cf/CalculatedFieldType.java | 2 +- .../data/cf/configuration/Argument.java | 1 + .../BaseCalculatedFieldConfiguration.java | 16 +++- .../CalculatedFieldConfiguration.java | 3 +- ...stRecordsCalculatedFieldConfiguration.java | 39 ++++++++ .../dao/model/sql/CalculatedFieldEntity.java | 16 ++-- ...efaultNativeCalculatedFieldRepository.java | 14 ++- 14 files changed, 323 insertions(+), 48 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 9edea50a05..a985f093c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -48,12 +48,16 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -69,7 +73,10 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.CalculationContext; +import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; @@ -84,6 +91,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -109,6 +117,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private static final int MAX_LAST_RECORDS_VALUE = 1024; + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -215,7 +225,19 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); - updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); + Map argumentValues = updatedTelemetry.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + argumentEntry.setKvEntry(entry.getValue()); + if (entry.getValue() instanceof TsKvEntry) { + argumentEntry.setKvEntries(List.of((TsKvEntry) entry.getValue())); + } + return argumentEntry; + } + )); + updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); @@ -308,12 +330,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); + Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument, entityId), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, argument), new FutureCallback<>() { @Override - public void onSuccess(Optional result) { - argumentValues.put(key, result.orElse(null)); + public void onSuccess(ArgumentEntry result) { + argumentValues.put(key, result); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -327,10 +349,37 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument, EntityId targetEntityId) { + private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, Argument argument) { + TenantId tenantId = calculatedField.getTenantId(); + EntityId cfEntityId = calculatedField.getEntityId(); EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - return switch (argument.getType()) { + EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) + ? cfEntityId + : argumentEntityId; + if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { + return fetchLastRecords(tenantId, entityId, argument); + } + return fetchKvEntry(tenantId, entityId, argument); + } + + private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { + long startTs = Math.max(argument.getStartTs(), 0); + long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); + long endTs = startTs + timeWindow; + int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + + return Futures.transform(lastRecordsFuture, lastRecords -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + argumentEntry.setKvEntries(lastRecords); + return argumentEntry; + }, calculatedFieldExecutor); + } + + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + ListenableFuture> kvEntryFuture = switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), result -> result.or(() -> Optional.of( @@ -342,9 +391,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas result -> result.or(() -> Optional.of( new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) )), - MoreExecutors.directExecutor()); + calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; + return Futures.transform(kvEntryFuture, kvEntry -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + if (kvEntry.isPresent()) { + argumentEntry.setKvEntry(kvEntry.orElse(null)); + } + return argumentEntry; + }, calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -359,7 +415,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -373,7 +429,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - ListenableFuture resultFuture = state.performCalculation(calculatedField.getTenantId(), calculatedField.getConfiguration(), tbelInvokeService); + CalculationContext ctx = CalculationContext.builder() + .tenantId(calculatedField.getTenantId()) + .configuration(calculatedField.getConfiguration()) + .tbelInvokeService(tbelInvokeService) + .build(); + ListenableFuture resultFuture = state.performCalculation(ctx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { @@ -414,6 +475,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return switch (calculatedFieldType) { case SIMPLE -> new SimpleCalculatedFieldState(); case SCRIPT -> new ScriptCalculatedFieldState(); + case LAST_RECORDS -> new LastRecordsCalculatedFieldState(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java new file mode 100644 index 0000000000..29e3417bde --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +public class ArgumentEntry { + + private KvEntry kvEntry; + private List kvEntries; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index dffcf09820..e4c4440921 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -19,11 +19,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -36,7 +33,8 @@ import java.util.Map; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT") + @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), + @JsonSubTypes.Type(value = LastRecordsCalculatedFieldState.class, name = "LAST_RECORDS") }) public interface CalculatedFieldState { @@ -47,8 +45,8 @@ public interface CalculatedFieldState { return argumentValues.keySet().containsAll(arguments.keySet()); } - void initState(Map argumentValues); + void initState(Map argumentValues); - ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService); + ListenableFuture performCalculation(CalculationContext ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java new file mode 100644 index 0000000000..656763ea48 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.Map; + +@Data +@Builder +public class CalculationContext { + + private TenantId tenantId; + private CalculatedFieldConfiguration configuration; + private TbelInvokeService tbelInvokeService; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java new file mode 100644 index 0000000000..2f26d71f91 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Data +public class LastRecordsCalculatedFieldState implements CalculatedFieldState { + + private Map> arguments; + + public LastRecordsCalculatedFieldState() { + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LAST_RECORDS; + } + + + @Override + public void initState(Map argumentValues) { + if (arguments == null) { + arguments = new HashMap<>(); + } + argumentValues.forEach((key, argumentEntry) -> { + List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); + tsKvEntryList.addAll(argumentEntry.getKvEntries()); + }); + } + + + @Override + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration configuration = ctx.getConfiguration(); + Map configArguments = configuration.getArguments(); + Output output = configuration.getOutput(); + + Map resultMap = new HashMap<>(); + + arguments.replaceAll((key, entries) -> { + int limit = configArguments.get(key).getLimit(); + List limitedEntries = entries.stream() + .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) + .limit(limit) + .collect(Collectors.toList()); + + Map valueWithTs = limitedEntries.stream() + .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); + resultMap.put(key, valueWithTs); + + return limitedEntries; + }); + + CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); + calculatedFieldResult.setType(output.getType()); + calculatedFieldResult.setScope(output.getScope()); + calculatedFieldResult.setResultMap(resultMap); + + return Futures.immediateFuture(calculatedFieldResult); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 5c984a8d16..0e9b00ad7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -40,7 +40,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @JsonIgnore private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - private Map arguments = new HashMap<>(); + private Map arguments; public ScriptCalculatedFieldState() { } @@ -50,22 +50,27 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { return CalculatedFieldType.SCRIPT; } + @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { - this.arguments = new HashMap<>(); + arguments = new HashMap<>(); } - this.arguments.putAll(argumentValues); + argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); } + @Override - public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); + TbelInvokeService tbelInvokeService = ctx.getTbelInvokeService(); + if (tbelInvokeService == null) { throw new IllegalArgumentException("TBEL script engine is disabled!"); } if (calculatedFieldScriptEngine == null) { - initEngine(tenantId, calculatedFieldConfiguration, tbelInvokeService); + initEngine(ctx.getTenantId(), calculatedFieldConfiguration, tbelInvokeService); } ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 08c4741de5..d3d3b5f636 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -20,12 +20,10 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -37,21 +35,26 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { private Map arguments; + public SimpleCalculatedFieldState() { + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } - arguments.putAll(argumentValues); + argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); } @Override - public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); + Output output = calculatedFieldConfiguration.getOutput(); Map arguments = calculatedFieldConfiguration.getArguments(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 89173b35b9..63b6d8d1dd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -17,6 +17,6 @@ package org.thingsboard.server.common.data.cf; public enum CalculatedFieldType { - SIMPLE, SCRIPT + SIMPLE, SCRIPT, LAST_RECORDS } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index f34f5e9cb7..0d70591a38 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -29,6 +29,7 @@ public class Argument { private String defaultValue; private int limit; + private long startTs; private long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 55a43a00ce..f7cc53b7cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -101,6 +101,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("type", argument.getType()); argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); + argumentNode.put("limit", String.valueOf(argument.getLimit())); + argumentNode.put("startTs", String.valueOf(argument.getStartTs())); + argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); }); if (expression != null) { @@ -144,7 +147,18 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { argument.setScope(AttributeScope.valueOf(scope.asText())); } - argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + if (argumentNode.hasNonNull("defaultValue")) { + argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + } + if (argumentNode.hasNonNull("limit")) { + argument.setLimit(argumentNode.get("limit").asInt()); + } + if (argumentNode.hasNonNull("startTs")) { + argument.setStartTs(argumentNode.get("startTs").asLong()); + } + if (argumentNode.hasNonNull("timeWindow")) { + argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); + } arguments.put(key, argument); }); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 5c428bd628..15f7a82c40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -35,7 +35,8 @@ import java.util.UUID; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), + @JsonSubTypes.Type(value = LastRecordsCalculatedFieldConfiguration.class, name = "LAST_RECORDS") }) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..c3f5804227 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.UUID; + +@Data +public class LastRecordsCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public LastRecordsCalculatedFieldConfiguration() { + } + + public LastRecordsCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LAST_RECORDS; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 6500d2a1e7..b06676f70b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,8 +24,9 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -120,14 +121,11 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - switch (CalculatedFieldType.valueOf(type)) { - case SIMPLE: - return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT: - return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - default: - throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); - } + return switch (CalculatedFieldType.valueOf(type)) { + case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); + case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); + }; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index fc40f72c93..2acd4d75c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -136,14 +137,11 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { - switch (type) { - case SIMPLE: - return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT: - return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - default: - throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); - } + return switch (type) { + case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); + case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); + }; } } From 80d9b22068ea44a5fc3d4bada2baa5c704d24246 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 26 Nov 2024 11:31:49 +0200 Subject: [PATCH 037/416] added argument entry interface --- ...efaultCalculatedFieldExecutionService.java | 66 +++++++------------ .../service/cf/ctx/state/ArgumentEntry.java | 19 ++++-- .../ctx/state/BaseCalculatedFieldState.java | 48 ++++++++++++++ .../cf/ctx/state/CalculationContext.java | 6 +- .../service/cf/ctx/state/KvArgumentEntry.java | 31 +++++++++ .../ctx/state/LastRecordsArgumentEntry.java | 33 ++++++++++ .../LastRecordsCalculatedFieldState.java | 21 ++---- .../ctx/state/ScriptCalculatedFieldState.java | 60 ++++++----------- .../ctx/state/SimpleCalculatedFieldState.java | 31 ++------- 9 files changed, 185 insertions(+), 130 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index a985f093c9..322443c848 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -191,19 +191,19 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(tenantId, cf, entityId, callback); + initializeStateForEntity(cf, entityId, callback); } case ASSET_PROFILE -> { log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(cf, assetId, callback)); } case DEVICE_PROFILE -> { log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(cf, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -226,17 +226,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - argumentEntry.setKvEntry(entry.getValue()); - if (entry.getValue() instanceof TsKvEntry) { - argumentEntry.setKvEntries(List.of((TsKvEntry) entry.getValue())); - } - return argumentEntry; - } - )); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createArgumentEntry(entry.getValue()))); updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { @@ -254,11 +244,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - List cfIdsOfOldProfile = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId); - cfIdsOfOldProfile.forEach(id -> states.remove(new CalculatedFieldCtxId(id.getId(), entityId.getId()))); - List ctxIdsToDelete = cfIdsOfOldProfile.stream().map(cfId -> JacksonUtil.writeValueAsString(new CalculatedFieldCtxId(cfId.getId(), entityId.getId()))).toList(); - rocksDBService.deleteAll(ctxIdsToDelete); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); @@ -269,7 +254,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) .stream() .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) - .forEach(cf -> initializeStateForEntity(tenantId, cf, entityId, callback)); + .forEach(cf -> initializeStateForEntity(cf, entityId, callback)); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -328,11 +313,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { + private void initializeStateForEntity(CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, entityId, argument), new FutureCallback<>() { @Override public void onSuccess(ArgumentEntry result) { argumentValues.put(key, result); @@ -349,12 +334,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, Argument argument) { + private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedField.getTenantId(); - EntityId cfEntityId = calculatedField.getEntityId(); EntityId argumentEntityId = argument.getEntityId(); EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) - ? cfEntityId + ? targetEntityId : argumentEntityId; if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { return fetchLastRecords(tenantId, entityId, argument); @@ -371,11 +355,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, lastRecords -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - argumentEntry.setKvEntries(lastRecords); - return argumentEntry; - }, calculatedFieldExecutor); + return Futures.transform(lastRecordsFuture, ArgumentEntry::createArgumentEntry, calculatedFieldExecutor); } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { @@ -394,13 +374,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - if (kvEntry.isPresent()) { - argumentEntry.setKvEntry(kvEntry.orElse(null)); - } - return argumentEntry; - }, calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createArgumentEntry(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -429,12 +403,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - CalculationContext ctx = CalculationContext.builder() - .tenantId(calculatedField.getTenantId()) - .configuration(calculatedField.getConfiguration()) - .tbelInvokeService(tbelInvokeService) - .build(); - ListenableFuture resultFuture = state.performCalculation(ctx); + ListenableFuture resultFuture = state.performCalculation(buildCalculationContext(calculatedField)); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { @@ -464,6 +433,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private CalculationContext buildCalculationContext(CalculatedField calculatedField) { + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + return CalculationContext.builder() + .tenantId(calculatedField.getTenantId()) + .arguments(configuration.getArguments()) + .output(configuration.getOutput()) + .expression(configuration.getExpression()) + .tbelInvokeService(tbelInvokeService) + .build(); + } + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 29e3417bde..3097056d11 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,16 +15,25 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Data; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; -@Data -public class ArgumentEntry { +public interface ArgumentEntry { - private KvEntry kvEntry; - private List kvEntries; + Object getValue(); + + static ArgumentEntry createArgumentEntry(KvEntry kvEntry) { + if (kvEntry instanceof TsKvEntry tsKvEntry) { + return new LastRecordsArgumentEntry(List.of(tsKvEntry)); + } else { + return new KvArgumentEntry(kvEntry); + } + } + + static ArgumentEntry createArgumentEntry(List kvEntries) { + return new LastRecordsArgumentEntry(kvEntries); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java new file mode 100644 index 0000000000..bac318a1b9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; + +public abstract class BaseCalculatedFieldState implements CalculatedFieldState { + + protected Map arguments; + + public BaseCalculatedFieldState() { + } + + @Override + public void initState(Map argumentValues) { + if (arguments == null) { + arguments = new HashMap<>(); + } +// argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); + } + + protected CalculatedFieldResult buildResult(Output output, Map resultMap) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.setType(output.getType()); + result.setScope(output.getScope()); + result.setResultMap(resultMap); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java index 656763ea48..aaabaec0d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java @@ -18,7 +18,9 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Builder; import lombok.Data; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; @@ -29,7 +31,9 @@ import java.util.Map; public class CalculationContext { private TenantId tenantId; - private CalculatedFieldConfiguration configuration; + private Map arguments; + private Output output; + private String expression; private TbelInvokeService tbelInvokeService; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java new file mode 100644 index 0000000000..0bd21d452e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.KvEntry; + +@Data +public class KvArgumentEntry implements ArgumentEntry { + + private final KvEntry kvEntry; + + @Override + public Object getValue() { + return kvEntry; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java new file mode 100644 index 0000000000..8729f022fa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +public class LastRecordsArgumentEntry implements ArgumentEntry { + + private final List kvEntries; + + @Override + public Object getValue() { + return kvEntries; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 2f26d71f91..0428d0823d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import aj.org.objectweb.asm.TypeReference; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; @@ -33,7 +35,7 @@ import java.util.Map; import java.util.stream.Collectors; @Data -public class LastRecordsCalculatedFieldState implements CalculatedFieldState { +public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { private Map> arguments; @@ -53,21 +55,16 @@ public class LastRecordsCalculatedFieldState implements CalculatedFieldState { } argumentValues.forEach((key, argumentEntry) -> { List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); - tsKvEntryList.addAll(argumentEntry.getKvEntries()); +// tsKvEntryList.addAll(argumentEntry.getKvEntries()); }); } @Override public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration configuration = ctx.getConfiguration(); - Map configArguments = configuration.getArguments(); - Output output = configuration.getOutput(); - Map resultMap = new HashMap<>(); - arguments.replaceAll((key, entries) -> { - int limit = configArguments.get(key).getLimit(); + int limit = ctx.getArguments().get(key).getLimit(); List limitedEntries = entries.stream() .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) .limit(limit) @@ -79,13 +76,7 @@ public class LastRecordsCalculatedFieldState implements CalculatedFieldState { return limitedEntries; }); - - CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); - calculatedFieldResult.setType(output.getType()); - calculatedFieldResult.setScope(output.getScope()); - calculatedFieldResult.setResultMap(resultMap); - - return Futures.immediateFuture(calculatedFieldResult); + return Futures.immediateFuture(buildResult(ctx.getOutput(), resultMap)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 0e9b00ad7d..047bfd0f8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -35,16 +35,11 @@ import java.util.Map; @Data @Slf4j -public class ScriptCalculatedFieldState implements CalculatedFieldState { +public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @JsonIgnore private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - private Map arguments; - - public ScriptCalculatedFieldState() { - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -52,50 +47,35 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new HashMap<>(); - } - argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } + public ListenableFuture performCalculation(CalculationContext ctx) { + if (isValid(this.arguments, ctx.getArguments())) { + if (calculatedFieldScriptEngine == null) { + initEngine(ctx.getTenantId(), ctx.getExpression(), ctx.getTbelInvokeService()); + } + ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(this.arguments); - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); - TbelInvokeService tbelInvokeService = ctx.getTbelInvokeService(); + return Futures.transform(resultFuture, result -> { + Map resultMap = result instanceof Map + ? JacksonUtil.convertValue(result, Map.class) + : new HashMap<>(); - if (tbelInvokeService == null) { - throw new IllegalArgumentException("TBEL script engine is disabled!"); + return buildResult(ctx.getOutput(), resultMap); + }, MoreExecutors.directExecutor()); } + return null; + } - if (calculatedFieldScriptEngine == null) { - initEngine(ctx.getTenantId(), calculatedFieldConfiguration, tbelInvokeService); + private void initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); } - ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); - - return Futures.transform(resultFuture, result -> { - Output output = calculatedFieldConfiguration.getOutput(); - Map resultMap = result instanceof Map - ? JacksonUtil.convertValue(result, Map.class) - : new HashMap<>(); - - CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); - calculatedFieldResult.setType(output.getType()); - calculatedFieldResult.setScope(output.getScope()); - calculatedFieldResult.setResultMap(resultMap); - - return calculatedFieldResult; - }, MoreExecutors.directExecutor()); - } - - private void initEngine(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( tenantId, tbelInvokeService, - calculatedFieldConfiguration.getExpression(), - arguments.keySet().toArray(new String[0]) + expression, + this.arguments.keySet().toArray(new String[0]) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index d3d3b5f636..fbdd1eb354 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -31,36 +31,17 @@ import java.util.HashMap; import java.util.Map; @Data -public class SimpleCalculatedFieldState implements CalculatedFieldState { - - private Map arguments; - - public SimpleCalculatedFieldState() { - } +public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } - @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new HashMap<>(); - } - argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } - @Override public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); - - Output output = calculatedFieldConfiguration.getOutput(); - Map arguments = calculatedFieldConfiguration.getArguments(); - - if (isValid(this.arguments, arguments)) { - CalculatedFieldResult result = new CalculatedFieldResult(); - String expression = calculatedFieldConfiguration.getExpression(); + if (isValid(this.arguments, ctx.getArguments())) { + String expression = ctx.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { @@ -76,10 +57,8 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { double expressionResult = expr.evaluate(); - result.setType(output.getType()); - result.setScope(output.getScope()); - result.setResultMap(Map.of(output.getName(), expressionResult)); - return Futures.immediateFuture(result); + Output output = ctx.getOutput(); + return Futures.immediateFuture(buildResult(output, Map.of(output.getName(), expressionResult))); } return null; } From d684c8777a2d7ec438e8a4b7b7de0a6841eaae49 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 26 Nov 2024 17:33:01 +0200 Subject: [PATCH 038/416] improved usage of calculation ctx --- .../cf/CalculatedFieldExecutionService.java | 2 +- .../service/cf/CalculatedFieldResult.java | 8 +- ...efaultCalculatedFieldExecutionService.java | 95 +++++++++---------- ...Ctx.java => CalculatedFieldEntityCtx.java} | 8 +- ...d.java => CalculatedFieldEntityCtxId.java} | 2 +- .../service/cf/ctx/state/ArgumentEntry.java | 26 +++-- ...KvArgumentEntry.java => ArgumentType.java} | 15 +-- .../ctx/state/BaseCalculatedFieldState.java | 30 +++--- .../cf/ctx/state/CalculatedFieldCtx.java | 72 ++++++++++++++ .../state/CalculatedFieldScriptEngine.java | 7 +- .../cf/ctx/state/CalculatedFieldState.java | 8 +- .../CalculatedFieldTbelScriptEngine.java | 18 +++- .../ctx/state/LastRecordsArgumentEntry.java | 16 +++- .../LastRecordsCalculatedFieldState.java | 46 +++++---- .../ctx/state/ScriptCalculatedFieldState.java | 47 ++------- .../ctx/state/SimpleCalculatedFieldState.java | 9 +- ...ext.java => SingleValueArgumentEntry.java} | 32 +++---- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../data/cf/configuration/Argument.java | 1 - .../BaseCalculatedFieldConfiguration.java | 4 - 20 files changed, 257 insertions(+), 191 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/{CalculatedFieldCtx.java => CalculatedFieldEntityCtx.java} (79%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/{CalculatedFieldCtxId.java => CalculatedFieldEntityCtxId.java} (90%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{KvArgumentEntry.java => ArgumentType.java} (72%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{CalculationContext.java => SingleValueArgumentEntry.java} (51%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6b7b12655f..6d7cf0e741 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,6 +29,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); - void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 87f1d08a84..1f8a06c8fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -21,13 +21,15 @@ import org.thingsboard.server.common.data.AttributeScope; import java.util.Map; @Data -public class CalculatedFieldResult { +public final class CalculatedFieldResult { private String type; private AttributeScope scope; private Map resultMap; - public CalculatedFieldResult() { + public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { + this.type = type; + this.scope = scope; + this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap); } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 322443c848..0d269f032f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -71,11 +71,11 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.CalculationContext; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; @@ -115,7 +115,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -188,22 +189,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = cf.getEntityId(); calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); + CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); + calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(cf, entityId, callback); + initializeStateForEntity(calculatedFieldCtx, entityId, callback); } case ASSET_PROFILE -> { log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(cf, assetId, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); } case DEVICE_PROFILE -> { log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(cf, deviceId, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(calculatedFieldCtx, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -224,10 +227,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { - CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { + CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); + return new CalculatedFieldCtx(calculatedField, tbelInvokeService); + }); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createArgumentEntry(entry.getValue()))); - updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + updateOrInitializeState(calculatedFieldCtx, calculatedFieldCtx.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); @@ -235,7 +241,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); @@ -246,15 +252,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { - CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); }); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) .stream() - .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) - .forEach(cf -> initializeStateForEntity(cf, entityId, callback)); + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -267,6 +273,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(newCalculatedField.getId(), callback); } else { calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + calculatedFieldsCtx.put(newCalculatedField.getId(), new CalculatedFieldCtx(newCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -277,6 +284,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); + calculatedFieldsCtx.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); List statesToRemove = states.keySet().stream() .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) @@ -309,20 +317,20 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } - private void initializeStateForEntity(CalculatedField calculatedField, EntityId entityId, TbCallback callback) { - Map arguments = calculatedField.getConfiguration().getArguments(); + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { + Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, entityId, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedFieldCtx, entityId, argument), new FutureCallback<>() { @Override public void onSuccess(ArgumentEntry result) { argumentValues.put(key, result); if (remaining.decrementAndGet() == 0) { - updateOrInitializeState(calculatedField, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); } } @@ -334,28 +342,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, EntityId targetEntityId, Argument argument) { - TenantId tenantId = calculatedField.getTenantId(); + private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { + TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { + if (CalculatedFieldType.LAST_RECORDS.equals(calculatedFieldCtx.getCfType())) { return fetchLastRecords(tenantId, entityId, argument); } return fetchKvEntry(tenantId, entityId, argument); } private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { - long startTs = Math.max(argument.getStartTs(), 0); + long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); - long endTs = startTs + timeWindow; + long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, ArgumentEntry::createArgumentEntry, calculatedFieldExecutor); + return Futures.transform(lastRecordsFuture, ArgumentEntry::createLastRecordsArgument, calculatedFieldExecutor); } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { @@ -374,7 +382,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createArgumentEntry(kvEntry.orElse(null)), calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -389,32 +397,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctx -> new CalculatedFieldEntityCtx(entityCtxId, null)); - CalculatedFieldState state = calculatedFieldCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); if (state == null) { - state = createStateByType(calculatedField.getType()); + state = createStateByType(calculatedFieldCtx.getCfType()); } state.initState(argumentValues); - calculatedFieldCtx.setState(state); - states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - ListenableFuture resultFuture = state.performCalculation(buildCalculationContext(calculatedField)); + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), entityId, result); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); } } @Override public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedField.getId(), entityId, t); + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); } }, MoreExecutors.directExecutor()); @@ -433,17 +441,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculationContext buildCalculationContext(CalculatedField calculatedField) { - CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); - return CalculationContext.builder() - .tenantId(calculatedField.getTenantId()) - .arguments(configuration.getArguments()) - .output(configuration.getOutput()) - .expression(configuration.getExpression()) - .tbelInvokeService(tbelInvokeService) - .build(); - } - private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java index 4b2a6c918f..7a8384b6bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java @@ -19,15 +19,15 @@ import lombok.Data; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data -public class CalculatedFieldCtx { +public class CalculatedFieldEntityCtx { - private CalculatedFieldCtxId id; + private CalculatedFieldEntityCtxId id; private CalculatedFieldState state; - public CalculatedFieldCtx() { + public CalculatedFieldEntityCtx() { } - public CalculatedFieldCtx(CalculatedFieldCtxId id, CalculatedFieldState state) { + public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { this.id = id; this.state = state; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index a316c54b76..f7c451efee 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -17,5 +17,5 @@ package org.thingsboard.server.service.cf.ctx; import java.util.UUID; -public record CalculatedFieldCtxId(UUID cfId, UUID entityId) { +public record CalculatedFieldEntityCtxId(UUID cfId, UUID entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 3097056d11..6218f38edf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,25 +15,35 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.stream.Collectors; +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), + @JsonSubTypes.Type(value = LastRecordsArgumentEntry.class, name = "LAST_RECORDS") +}) public interface ArgumentEntry { + ArgumentType getType(); + Object getValue(); - static ArgumentEntry createArgumentEntry(KvEntry kvEntry) { - if (kvEntry instanceof TsKvEntry tsKvEntry) { - return new LastRecordsArgumentEntry(List.of(tsKvEntry)); - } else { - return new KvArgumentEntry(kvEntry); - } + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { + return new SingleValueArgumentEntry(kvEntry.getValue()); } - static ArgumentEntry createArgumentEntry(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries); + static ArgumentEntry createLastRecordsArgument(List kvEntries) { + return new LastRecordsArgumentEntry(kvEntries.stream() .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java index 0bd21d452e..f2f0eac60d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java @@ -15,17 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Data; -import org.thingsboard.server.common.data.kv.KvEntry; - -@Data -public class KvArgumentEntry implements ArgumentEntry { - - private final KvEntry kvEntry; - - @Override - public Object getValue() { - return kvEntry; - } - +public enum ArgumentType { + SINGLE_VALUE, LAST_RECORDS } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index bac318a1b9..54df89e757 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,34 +15,36 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.service.cf.CalculatedFieldResult; - import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { - protected Map arguments; + protected Map arguments; public BaseCalculatedFieldState() { } + @Override + public Map getArguments() { + return this.arguments; + } + @Override public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } -// argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } - - protected CalculatedFieldResult buildResult(Output output, Map resultMap) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.setType(output.getType()); - result.setScope(output.getScope()); - result.setResultMap(resultMap); - return result; + arguments.putAll( + argumentValues.entrySet().stream() + .peek(entry -> { + if (entry.getValue() instanceof LastRecordsArgumentEntry) { + throw new IllegalArgumentException("Last records argument entry is not allowed for single calculated field state"); + } + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java new file mode 100644 index 0000000000..10395a0d5d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +@Data +public class CalculatedFieldCtx { + + private CalculatedFieldId cfId; + private TenantId tenantId; + private EntityId entityId; + private CalculatedFieldType cfType; + private Map arguments; + private Output output; + private String expression; + private TbelInvokeService tbelInvokeService; + private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + this.cfId = calculatedField.getId(); + this.tenantId = calculatedField.getTenantId(); + this.entityId = calculatedField.getEntityId(); + this.cfType = calculatedField.getType(); + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + this.arguments = configuration.getArguments(); + this.output = configuration.getOutput(); + this.expression = configuration.getExpression(); + this.tbelInvokeService = tbelInvokeService; + if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { + this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + } + } + + private CalculatedFieldScriptEngine initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + return new CalculatedFieldTbelScriptEngine( + tenantId, + tbelInvokeService, + expression, + arguments.keySet().toArray(new String[0]) + ); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 6b54536019..212d971398 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -16,13 +16,16 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.kv.KvEntry; import java.util.Map; public interface CalculatedFieldScriptEngine { - ListenableFuture executeScriptAsync(Map arguments); + ListenableFuture executeScriptAsync(Map arguments); + + ListenableFuture> executeToMapAsync(Map arguments); + + ListenableFuture> executeToMapTransform(Object result); void destroy(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index e4c4440921..f0882cbea4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -41,12 +41,14 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map argumentValues, Map arguments) { - return argumentValues.keySet().containsAll(arguments.keySet()); + Map getArguments(); + + default boolean isValid(Map arguments) { + return getArguments().keySet().containsAll(arguments.keySet()); } void initState(Map argumentValues); - ListenableFuture performCalculation(CalculationContext ctx); + ListenableFuture performCalculation(CalculatedFieldCtx ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 5cc58b2a95..363eefd44e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -52,9 +53,9 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture executeScriptAsync(Map arguments) { + public ListenableFuture executeScriptAsync(Map arguments) { log.trace("execute script async, arguments {}", arguments); - Object[] args = arguments.values().stream().map(KvEntry::getValue).toArray(); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { @@ -71,6 +72,19 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng }, MoreExecutors.directExecutor()); } + @Override + public ListenableFuture> executeToMapAsync(Map arguments) { + return Futures.transformAsync(executeScriptAsync(arguments), this::executeToMapTransform, MoreExecutors.directExecutor()); + } + + @Override + public ListenableFuture> executeToMapTransform(Object result) { + if (result instanceof Map) { + return Futures.immediateFuture((Map) result); + } + throw new IllegalArgumentException("Wrong result type: [" + result.getClass().getName() + "]"); + } + @Override public void destroy() { tbelInvokeService.release(this.scriptId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 8729f022fa..39da1838bc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -15,19 +15,27 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import lombok.NoArgsConstructor; -import java.util.List; +import java.util.Map; @Data +@NoArgsConstructor +@AllArgsConstructor public class LastRecordsArgumentEntry implements ArgumentEntry { - private final List kvEntries; + private Map tsRecords; + + @Override + public ArgumentType getType() { + return ArgumentType.LAST_RECORDS; + } @Override public Object getValue() { - return kvEntries; + return tsRecords.values(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 0428d0823d..437707bfdb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -15,14 +15,10 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import aj.org.objectweb.asm.TypeReference; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -37,8 +33,6 @@ import java.util.stream.Collectors; @Data public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { - private Map> arguments; - public LastRecordsCalculatedFieldState() { } @@ -47,36 +41,46 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { return CalculatedFieldType.LAST_RECORDS; } - @Override public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } argumentValues.forEach((key, argumentEntry) -> { - List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); -// tsKvEntryList.addAll(argumentEntry.getKvEntries()); + LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) + arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new HashMap<>())); + if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { + existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry + && singleValueArgumentEntry.getValue() instanceof TsKvEntry tsKvEntry) { + existingArgumentEntry.getTsRecords().put(tsKvEntry.getTs(), tsKvEntry.getValue()); + } }); } - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Map resultMap = new HashMap<>(); - arguments.replaceAll((key, entries) -> { + arguments.replaceAll((key, argumentEntry) -> { int limit = ctx.getArguments().get(key).getLimit(); - List limitedEntries = entries.stream() - .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) - .limit(limit) - .collect(Collectors.toList()); - Map valueWithTs = limitedEntries.stream() - .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); - resultMap.put(key, valueWithTs); + // TODO: implement removing if size > limit + + +// List limitedEntries = entries.stream() +// .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) +// .limit(limit) +// .collect(Collectors.toList()); +// +// Map valueWithTs = limitedEntries.stream() +// .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); +// resultMap.put(key, valueWithTs); - return limitedEntries; +// return new LastRecordsArgumentEntry(limitedEntries); + return null; }); - return Futures.immediateFuture(buildResult(ctx.getOutput(), resultMap)); + Output output = ctx.getOutput(); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 047bfd0f8c..16905930d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -15,68 +15,37 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; import java.util.Map; @Data @Slf4j public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { - @JsonIgnore - private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; } - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - if (isValid(this.arguments, ctx.getArguments())) { - if (calculatedFieldScriptEngine == null) { - initEngine(ctx.getTenantId(), ctx.getExpression(), ctx.getTbelInvokeService()); - } - - ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(this.arguments); - - return Futures.transform(resultFuture, result -> { - Map resultMap = result instanceof Map - ? JacksonUtil.convertValue(result, Map.class) - : new HashMap<>(); - - return buildResult(ctx.getOutput(), resultMap); - }, MoreExecutors.directExecutor()); + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + Output output = ctx.getOutput(); + if (isValid(ctx.getArguments())) { + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(this.arguments); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); } return null; } - private void initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { - if (tbelInvokeService == null) { - throw new IllegalArgumentException("TBEL script engine is disabled!"); - } - - calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( - tenantId, - tbelInvokeService, - expression, - this.arguments.keySet().toArray(new String[0]) - ); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index fbdd1eb354..64eb409d3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -22,7 +22,6 @@ import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -39,8 +38,8 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - if (isValid(this.arguments, ctx.getArguments())) { + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + if (isValid(ctx.getArguments())) { String expression = ctx.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); @@ -52,13 +51,13 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValueAsString()))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(((KvEntry) v.getValue()).getValueAsString()))); expr.setVariables(variables); double expressionResult = expr.evaluate(); Output output = ctx.getOutput(); - return Futures.immediateFuture(buildResult(output, Map.of(output.getName(), expressionResult))); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } return null; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java similarity index 51% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index aaabaec0d4..504d213748 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,25 +15,25 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Builder; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; - -import java.util.Map; +import lombok.NoArgsConstructor; @Data -@Builder -public class CalculationContext { +@NoArgsConstructor +@AllArgsConstructor +public class SingleValueArgumentEntry implements ArgumentEntry { + + private Object value; + + @Override + public ArgumentType getType() { + return ArgumentType.SINGLE_VALUE; + } - private TenantId tenantId; - private Map arguments; - private Output output; - private String expression; - private TbelInvokeService tbelInvokeService; + @Override + public Object getValue() { + return value; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 3caaab6613..392e2637d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -685,7 +685,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityTypeChanged(profileUpdateMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChanged(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index 0d70591a38..f34f5e9cb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -29,7 +29,6 @@ public class Argument { private String defaultValue; private int limit; - private long startTs; private long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index f7cc53b7cf..f311fb737b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -102,7 +102,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); - argumentNode.put("startTs", String.valueOf(argument.getStartTs())); argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); }); @@ -153,9 +152,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (argumentNode.hasNonNull("limit")) { argument.setLimit(argumentNode.get("limit").asInt()); } - if (argumentNode.hasNonNull("startTs")) { - argument.setStartTs(argumentNode.get("startTs").asLong()); - } if (argumentNode.hasNonNull("timeWindow")) { argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); } From f9db64a14d036cbd6a5ffdbe3a733b177e9d081b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 27 Nov 2024 12:10:04 +0200 Subject: [PATCH 039/416] added deletion from map if ts out of timewindow --- .../service/cf/ctx/state/ArgumentEntry.java | 8 +++- .../ctx/state/LastRecordsArgumentEntry.java | 6 ++- .../LastRecordsCalculatedFieldState.java | 46 ++++++++----------- .../ctx/state/SimpleCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 20 ++++++-- 5 files changed, 46 insertions(+), 38 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 6218f38edf..6a62c0b1e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.TreeMap; import java.util.stream.Collectors; @JsonTypeInfo( @@ -34,16 +36,18 @@ import java.util.stream.Collectors; }) public interface ArgumentEntry { + @JsonIgnore ArgumentType getType(); Object getValue(); static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { - return new SingleValueArgumentEntry(kvEntry.getValue()); + return new SingleValueArgumentEntry(kvEntry); } static ArgumentEntry createLastRecordsArgument(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries.stream() .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue))); + return new LastRecordsArgumentEntry(kvEntries.stream(). + collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 39da1838bc..93fabd3cb8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -15,24 +15,26 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.Map; +import java.util.TreeMap; @Data @NoArgsConstructor @AllArgsConstructor public class LastRecordsArgumentEntry implements ArgumentEntry { - private Map tsRecords; + private TreeMap tsRecords; @Override public ArgumentType getType() { return ArgumentType.LAST_RECORDS; } + @JsonIgnore @Override public Object getValue() { return tsRecords.values(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 437707bfdb..dd69790236 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -19,16 +19,13 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.ArrayList; -import java.util.Comparator; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.TreeMap; @Data public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @@ -44,16 +41,15 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public void initState(Map argumentValues) { if (arguments == null) { - arguments = new HashMap<>(); + arguments = new TreeMap<>(); } argumentValues.forEach((key, argumentEntry) -> { LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) - arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new HashMap<>())); + arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new TreeMap<>())); if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry - && singleValueArgumentEntry.getValue() instanceof TsKvEntry tsKvEntry) { - existingArgumentEntry.getTsRecords().put(tsKvEntry.getTs(), tsKvEntry.getValue()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + existingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); } }); } @@ -61,26 +57,22 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Map resultMap = new HashMap<>(); - arguments.replaceAll((key, argumentEntry) -> { - int limit = ctx.getArguments().get(key).getLimit(); - - // TODO: implement removing if size > limit - - -// List limitedEntries = entries.stream() -// .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) -// .limit(limit) -// .collect(Collectors.toList()); -// -// Map valueWithTs = limitedEntries.stream() -// .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); -// resultMap.put(key, valueWithTs); - -// return new LastRecordsArgumentEntry(limitedEntries); - return null; + arguments.forEach((key, argumentEntry) -> { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + long necessaryIntervalTs = calculateIntervalStart(System.currentTimeMillis(), argument.getTimeWindow()); + tsRecords.entrySet().removeIf(tsRecord -> calculateIntervalStart(tsRecord.getKey(), argument.getTimeWindow()) < necessaryIntervalTs); + resultMap.put(key, tsRecords); }); Output output = ctx.getOutput(); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); } + private long calculateIntervalStart(long ts, long interval) { + return (ts / interval) * interval; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 64eb409d3a..fc97141806 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -21,9 +21,7 @@ import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -51,7 +49,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(((KvEntry) v.getValue()).getValueAsString()))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); expr.setVariables(variables); double expressionResult = expr.evaluate(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 504d213748..e0db8c50fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,17 +15,29 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.AllArgsConstructor; import lombok.Data; -import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; @Data -@NoArgsConstructor -@AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + private long ts; private Object value; + public SingleValueArgumentEntry() { + } + + public SingleValueArgumentEntry(KvEntry entry) { + if (entry instanceof TsKvEntry) { + this.ts = ((TsKvEntry) entry).getTs(); + } else if (entry instanceof AttributeKvEntry) { + this.ts = ((AttributeKvEntry) entry).getLastUpdateTs(); + } + this.value = entry.getValue(); + } + @Override public ArgumentType getType() { return ArgumentType.SINGLE_VALUE; From 2080b439a7e207ff186202c615f4cc161c010076 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 28 Nov 2024 11:28:51 +0200 Subject: [PATCH 040/416] added entity added/deleted events handling --- .../cf/CalculatedFieldExecutionService.java | 4 ++ ...efaultCalculatedFieldExecutionService.java | 46 ++++++++++++-- .../entitiy/EntityStateSourcingListener.java | 6 +- .../queue/DefaultTbClusterService.java | 61 +++++++++++++++++-- .../queue/DefaultTbCoreConsumerService.java | 27 ++++++++ .../server/cluster/TbClusterService.java | 2 + common/proto/src/main/proto/queue.proto | 13 ++++ .../server/dao/asset/BaseAssetService.java | 2 +- 8 files changed, 147 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6d7cf0e741..a1202ec74b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -31,4 +31,8 @@ public interface CalculatedFieldExecutionService { void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback); + + void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0d269f032f..3bb9be065d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -74,8 +74,8 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; @@ -227,6 +227,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { + log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); return new CalculatedFieldCtx(calculatedField, tbelInvokeService); @@ -247,7 +248,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) @@ -257,10 +257,44 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); }); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) - .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) - .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); + } + } + + @Override + public void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); + log.info("Received EntityCreateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + + initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); + } + } + + private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) + .stream() + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + } + + @Override + public void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback) { + try { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + log.info("Received EntityDeleteMsg for processing: entityId=[{}]", entityId); + List statesToRemove = states.keySet().stream() + .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 154f5f4833..57293169a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -146,7 +146,11 @@ public class EntityStateSourcingListener { log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event); switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { + case ASSET -> { + Asset asset = (Asset) event.getEntity(); + tbClusterService.onAssetDeleted(tenantId, asset, null); + } + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 58a16d9c0a..78a1f27a55 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -388,11 +388,26 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); + handleEntityDelete(tenantId, deviceId, device.getDeviceProfileId()); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); } + @Override + public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { + AssetId assetId = asset.getId(); + handleEntityDelete(tenantId, assetId, asset.getAssetProfileId()); + broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); + } + + private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); + if (cfExistsByProfile) { + sendEntityDeletedEvent(tenantId, entityId); + } + } + @Override public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { onDeviceDeleted(oldTenantId, device, null); @@ -613,11 +628,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - handleProfileChange(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + handleProfileUpdate(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } + } else { + handleEntityCreate(device.getTenantId(), device.getId(), device.getDeviceProfileId()); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -631,16 +648,26 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleProfileChange(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + handleProfileUpdate(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } + } else { + handleEntityCreate(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } - private void handleProfileChange(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + private void handleProfileUpdate(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + boolean cfExistsByOldProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + boolean cfExistsByNewProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, newProfileId); + if (cfExistsByOldProfile || cfExistsByNewProfile) { + sendEntityProfileUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + } + } + + private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityTypeUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + sendEntityAddedEvent(tenantId, entityId, profileId); } } @@ -801,7 +828,7 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } - private void sendEntityTypeUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + private void sendEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -817,4 +844,26 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } + private void sendEntityAddedEvent(TenantId tenantId, EntityId entityId, EntityId profileId) { + TransportProtos.EntityAddMsgProto.Builder builder = TransportProtos.EntityAddMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setEntityProfileType(profileId.getEntityType().name()); + builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); + builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); + TransportProtos.EntityAddMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityAddMsg(msg).build(), null); + } + + private void sendEntityDeletedEvent(TenantId tenantId, EntityId entityId) { + TransportProtos.EntityDeleteMsg.Builder builder = TransportProtos.EntityDeleteMsg.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityDeleteMsg(builder).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 392e2637d1..d477c6d753 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -320,6 +320,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityAdded(entityCreateMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process entity create message for entityId [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } + + private void forwardToCalculatedFieldService(TransportProtos.EntityDeleteMsg entityDeleteMsg, TbCallback callback) { + var entityId = EntityIdFactory.getByTypeAndUuid(entityDeleteMsg.getEntityType(), new UUID(entityDeleteMsg.getEntityIdMSB(), entityDeleteMsg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityDeleted(entityDeleteMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("Failed to process entity delete message for entity [{}]", entityId, t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 131b50a52c..69334b774f 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -100,6 +100,8 @@ public interface TbClusterService extends TbQueueClusterService { void onAssetUpdated(Asset asset, Asset old); + void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback); + void onResourceChange(TbResourceInfo resource, TbQueueCallback callback); void onResourceDeleted(TbResourceInfo resource, TbQueueCallback callback); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index bf49a68c80..b1ca9b5e56 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -794,6 +794,17 @@ message EntityProfileUpdateMsgProto { int64 newProfileIdLSB = 10; } +message EntityAddMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + string entityProfileType = 6; + int64 profileIdMSB = 7; + int64 profileIdLSB = 8; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1527,6 +1538,8 @@ message ToCoreMsg { DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; + EntityAddMsgProto entityAddMsg = 55; + EntityDeleteMsg entityDeleteMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 7242cfde68..564da127ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -239,7 +239,7 @@ public class BaseAssetService extends AbstractCachedEntityService Date: Thu, 28 Nov 2024 16:07:55 +0200 Subject: [PATCH 041/416] implemented update telemetry for entity by profile --- .../cf/CalculatedFieldExecutionService.java | 3 +- ...efaultCalculatedFieldExecutionService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 61 +++++++++++++------ 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index a1202ec74b..b605ee909e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -27,7 +28,7 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 3bb9be065d..d821f632fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -225,7 +225,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { @@ -234,7 +234,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(calculatedFieldCtx, calculatedFieldCtx.getEntityId(), argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 76c1383f6a..f6c19eb078 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -33,8 +33,10 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -56,6 +58,8 @@ import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import java.util.ArrayList; @@ -85,6 +89,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbApiUsageStateService apiUsageStateService; private final CalculatedFieldService calculatedFieldService; private final CalculatedFieldExecutionService calculatedFieldExecutionService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; private ExecutorService tsCallBackExecutor; @@ -97,7 +103,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, CalculatedFieldService calculatedFieldService, - CalculatedFieldExecutionService calculatedFieldExecutionService) { + CalculatedFieldExecutionService calculatedFieldExecutionService, + TbAssetProfileCache assetProfileCache, + TbDeviceProfileCache deviceProfileCache) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; @@ -105,6 +113,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer this.apiUsageStateService = apiUsageStateService; this.calculatedFieldService = calculatedFieldService; this.calculatedFieldExecutionService = calculatedFieldExecutionService; + this.assetProfileCache = assetProfileCache; + this.deviceProfileCache = deviceProfileCache; } @PostConstruct @@ -201,8 +211,17 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { - if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { - List cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + EntityType entityType = entityId.getEntityType(); + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType)) { + EntityId profileId; + if (EntityType.ASSET.equals(entityType)) { + profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + } else { + profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + } + List cfLinks = new ArrayList<>(); + cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId)); if (!cfLinks.isEmpty()) { cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); @@ -211,34 +230,36 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer Map updatedTelemetry = telemetry.stream() .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) .collect(Collectors.toMap( - entry -> { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - }, + entry -> getMappedKey(entry, attributes, timeSeries), entry -> entry, (v1, v2) -> v1 )); if (!updatedTelemetry.isEmpty()) { - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); } }); } } } + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + } + private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), From b910760b05c8597b1e44853f4eb928c7560b0865 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 29 Nov 2024 12:29:41 +0200 Subject: [PATCH 042/416] improved fetching cfs and states from db --- ...efaultCalculatedFieldExecutionService.java | 51 +++++++++---------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d821f632fe..2147e0a1b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,7 +36,6 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; @@ -81,7 +80,6 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -114,7 +112,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); @@ -131,7 +128,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(this::fetchCalculatedFields); } @PreDestroy @@ -176,7 +172,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + CalculatedField cf = getOfFetchFromDb(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -184,11 +180,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return; } } - List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { EntityId entityId = cf.getEntityId(); - calculatedFields.put(calculatedFieldId, cf); - calculatedFieldLinks.put(calculatedFieldId, links); CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); switch (entityId.getEntityType()) { @@ -229,7 +222,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { - CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); + CalculatedField calculatedField = getOfFetchFromDb(tenantId, id); return new CalculatedFieldCtx(calculatedField, tbelInvokeService); }); Map argumentValues = updatedTelemetry.entrySet().stream() @@ -300,14 +293,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = getOfFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { - onCalculatedFieldDelete(newCalculatedField.getId(), callback); + if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { + onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { - calculatedFields.put(newCalculatedField.getId(), newCalculatedField); - calculatedFieldsCtx.put(newCalculatedField.getId(), new CalculatedFieldCtx(newCalculatedField, tbelInvokeService)); + calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); + calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -316,12 +309,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { - calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); calculatedFieldsCtx.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() - .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) .map(JacksonUtil::writeValueAsString) .toList(); rocksDBService.deleteAll(statesToRemove); @@ -331,6 +323,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private CalculatedField getOfFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); + } + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -346,15 +342,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); - } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); @@ -433,7 +420,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctx -> new CalculatedFieldEntityCtx(entityCtxId, null)); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -462,6 +449,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, null); + } + return JacksonUtil.fromString(rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)), CalculatedFieldEntityCtx.class); + } + private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); From 894f4ead1b120d46809e6d03c326fa195fd69b0e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 29 Nov 2024 17:31:56 +0200 Subject: [PATCH 043/416] implemented logic not to fetch tenant/customer telemetry for each field --- ...efaultCalculatedFieldExecutionService.java | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 2147e0a1b7..732d050a22 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfig import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -80,6 +81,7 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -115,6 +117,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); + private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); + private static final int MAX_LAST_RECORDS_VALUE = 1024; @Value("${calculatedField.initFetchPackSize:50000}") @@ -366,6 +371,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); + if (EntityType.TENANT.equals(argumentEntityId.getEntityType()) || EntityType.CUSTOMER.equals(argumentEntityId.getEntityType())) { + return fetchFromStorage(tenantId, argumentEntityId, argument); + } EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; @@ -375,6 +383,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return fetchKvEntry(tenantId, entityId, argument); } + private ListenableFuture fetchFromStorage(TenantId tenantId, EntityId entityId, Argument argument) { + List kvEntries; + if (EntityType.TENANT.equals(entityId.getEntityType())) { + kvEntries = tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + } else { + kvEntries = customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + } + for (KvEntry kvEntry : kvEntries) { + if (kvEntry.getKey().equals(argument.getKey())) { + return Futures.immediateFuture(ArgumentEntry.createSingleValueArgument(kvEntry)); + } + } + return fetchKvEntry(tenantId, entityId, argument); + } + private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); @@ -403,7 +426,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> { + if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { + updateStorage(tenantId, entityId, kvEntry); + } + return ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)); + }, calculatedFieldExecutor); + } + + private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { + if (kvEntry.isEmpty()) { + return; + } + List kvEntries = switch (entityId.getEntityType()) { + case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + default -> null; + }; + if (kvEntries != null && !kvEntries.contains(kvEntry.get())) { + kvEntries.add(kvEntry.get()); + } } private KvEntry createDefaultKvEntry(Argument argument) { From 19234df2b8e03d8d2af392faa32dfb44cc71bc77 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 5 Dec 2024 16:46:12 +0200 Subject: [PATCH 044/416] improved implementation of telemetry update --- ...efaultCalculatedFieldExecutionService.java | 60 ++++++++++++------- .../DefaultTelemetrySubscriptionService.java | 11 ++-- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 732d050a22..4a1e987687 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -177,7 +177,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = getOfFetchFromDb(tenantId, calculatedFieldId); + CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -226,14 +226,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { - CalculatedField calculatedField = getOfFetchFromDb(tenantId, id); - return new CalculatedFieldCtx(calculatedField, tbelInvokeService); - }); + CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> { + if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { + updateStorage(tenantId, entityId, Optional.of(entry.getValue())); + } + return ArgumentEntry.createSingleValueArgument(entry.getValue()); + })); + + EntityId cfEntityId = calculatedField.getEntityId(); + switch (cfEntityId.getEntityType()) { + case ASSET_PROFILE, DEVICE_PROFILE -> { + boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); + if (isCommonEntity) { + PageDataIterable entities = cfEntityId.getEntityType() == EntityType.ASSET_PROFILE + ? new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) cfEntityId, pageLink), initFetchPackSize) + : new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) cfEntityId, pageLink), initFetchPackSize); + entities.forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + } else { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + } + } + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + } + log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); } @@ -299,7 +317,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = getOfFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); @@ -328,7 +346,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculatedField getOfFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + private CalculatedField getOrFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); } @@ -435,17 +453,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { - if (kvEntry.isEmpty()) { - return; - } - List kvEntries = switch (entityId.getEntityType()) { - case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - default -> null; - }; - if (kvEntries != null && !kvEntries.contains(kvEntry.get())) { - kvEntries.add(kvEntry.get()); - } + kvEntry.ifPresent(entry -> { + List kvEntries = switch (entityId.getEntityType()) { + case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + default -> null; + }; + if (kvEntries != null) { + kvEntries.removeIf(existingEntry -> existingEntry.getKey().equals(entry.getKey())); + kvEntries.add(entry); + } + }); } private KvEntry createDefaultKvEntry(Argument argument) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f6c19eb078..ad64204e0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -212,16 +212,15 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType)) { - EntityId profileId; + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { + EntityId profileId = null; if (EntityType.ASSET.equals(entityType)) { profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else { + } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = new ArrayList<>(); - cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId)); + List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); if (!cfLinks.isEmpty()) { cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); From dfeb0a7dbaf30b28529e27449f0fac9015f066f8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 6 Dec 2024 11:11:24 +0200 Subject: [PATCH 045/416] made executeScrioptAsync method flexible --- .../cf/ctx/state/CalculatedFieldScriptEngine.java | 4 ++-- .../cf/ctx/state/CalculatedFieldTbelScriptEngine.java | 11 ++++------- .../cf/ctx/state/ScriptCalculatedFieldState.java | 3 ++- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 212d971398..779f52c5d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -21,9 +21,9 @@ import java.util.Map; public interface CalculatedFieldScriptEngine { - ListenableFuture executeScriptAsync(Map arguments); + ListenableFuture executeScriptAsync(Object[] args); - ListenableFuture> executeToMapAsync(Map arguments); + ListenableFuture> executeToMapAsync(Object[] args); ListenableFuture> executeToMapTransform(Object result); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 363eefd44e..7ac032573f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,11 +19,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import javax.script.ScriptException; import java.util.Map; @@ -53,9 +51,8 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture executeScriptAsync(Map arguments) { - log.trace("execute script async, arguments {}", arguments); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + public ListenableFuture executeScriptAsync(Object[] args) { + log.trace("Executing script async, args {}", args); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { @@ -73,8 +70,8 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture> executeToMapAsync(Map arguments) { - return Futures.transformAsync(executeScriptAsync(arguments), this::executeToMapTransform, MoreExecutors.directExecutor()); + public ListenableFuture> executeToMapAsync(Object[] args) { + return Futures.transformAsync(executeScriptAsync(args), this::executeToMapTransform, MoreExecutors.directExecutor()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 16905930d3..99befa6e65 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -39,7 +39,8 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Output output = ctx.getOutput(); if (isValid(ctx.getArguments())) { - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(this.arguments); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); return Futures.transform(resultFuture, result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), MoreExecutors.directExecutor() From 303851015035cc3715190b5fc708ad6da173e807 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 9 Dec 2024 11:04:05 +0200 Subject: [PATCH 046/416] added implementation for handling profile entities --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 96 ++++++++++--------- .../queue/DefaultTbClusterService.java | 22 ++--- .../queue/DefaultTbCoreConsumerService.java | 27 ++---- common/proto/src/main/proto/queue.proto | 7 +- 5 files changed, 71 insertions(+), 85 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index b605ee909e..5a85529f6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -32,8 +32,6 @@ public interface CalculatedFieldExecutionService { void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); - void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback); - - void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback); + void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 4a1e987687..7ded80d013 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,11 +39,9 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -83,6 +81,7 @@ import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -117,6 +116,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); @@ -194,17 +194,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); initializeStateForEntity(calculatedFieldCtx, entityId, callback); } - case ASSET_PROFILE -> { - log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); - PageDataIterable assetIds = new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); - } - case DEVICE_PROFILE -> { - log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); - PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(calculatedFieldCtx, deviceId, callback)); + case ASSET_PROFILE, DEVICE_PROFILE -> { + log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -241,10 +233,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - PageDataIterable entities = cfEntityId.getEntityType() == EntityType.ASSET_PROFILE - ? new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) cfEntityId, pageLink), initFetchPackSize) - : new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) cfEntityId, pageLink), initFetchPackSize); - entities.forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); } else { updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); } @@ -266,6 +255,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.get(oldProfileId).remove(entityId); + profileEntities.computeIfAbsent(newProfileId, id -> new HashSet<>()).add(entityId); + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -280,39 +272,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback) { + public void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - log.info("Received EntityCreateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - - initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); - } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); - } - } - - private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) - .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) - .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); - } - - @Override - public void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback) { - try { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received EntityDeleteMsg for processing: entityId=[{}]", entityId); - List statesToRemove = states.keySet().stream() - .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); - rocksDBService.deleteAll(statesToRemove); + log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + if (proto.getDeleted()) { + log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.get(profileId).remove(entityId); + List statesToRemove = states.keySet().stream() + .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); + rocksDBService.deleteAll(statesToRemove); + } else { + log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); + initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + } } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); + log.trace("Failed to process profile entity msg: [{}]", proto, e); } } @@ -350,6 +331,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); } + private Set getOrFetchFromDBProfileEntities(TenantId tenantId, EntityId entityProfileId) { + return switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -365,6 +364,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } + private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) + .stream() + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + } + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 78a1f27a55..a706f943c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -404,7 +404,7 @@ public class DefaultTbClusterService implements TbClusterService { private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityDeletedEvent(tenantId, entityId); + sendProfileEntityEvent(tenantId, entityId, profileId, false, true); } } @@ -667,7 +667,7 @@ public class DefaultTbClusterService implements TbClusterService { private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityAddedEvent(tenantId, entityId, profileId); + sendProfileEntityEvent(tenantId, entityId, profileId, true, false); } } @@ -844,8 +844,8 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } - private void sendEntityAddedEvent(TenantId tenantId, EntityId entityId, EntityId profileId) { - TransportProtos.EntityAddMsgProto.Builder builder = TransportProtos.EntityAddMsgProto.newBuilder(); + private void sendProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { + TransportProtos.ProfileEntityMsgProto.Builder builder = TransportProtos.ProfileEntityMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setEntityType(entityId.getEntityType().name()); @@ -854,16 +854,10 @@ public class DefaultTbClusterService implements TbClusterService { builder.setEntityProfileType(profileId.getEntityType().name()); builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); - TransportProtos.EntityAddMsgProto msg = builder.build(); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityAddMsg(msg).build(), null); - } - - private void sendEntityDeletedEvent(TenantId tenantId, EntityId entityId) { - TransportProtos.EntityDeleteMsg.Builder builder = TransportProtos.EntityDeleteMsg.newBuilder(); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityDeleteMsg(builder).build(), null); + builder.setAdded(added); + builder.setDeleted(deleted); + TransportProtos.ProfileEntityMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setProfileEntityMsg(msg).build(), null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index d477c6d753..6cf42cb893 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -320,10 +320,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityAdded(entityCreateMsg, callback)); + private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { + var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { - log.warn("[{}] Failed to process entity create message for entityId [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(TransportProtos.EntityDeleteMsg entityDeleteMsg, TbCallback callback) { - var entityId = EntityIdFactory.getByTypeAndUuid(entityDeleteMsg.getEntityType(), new UUID(entityDeleteMsg.getEntityIdMSB(), entityDeleteMsg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityDeleted(entityDeleteMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("Failed to process entity delete message for entity [{}]", entityId, t); + log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); callback.onFailure(t); }); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b1ca9b5e56..76da257b32 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -794,7 +794,7 @@ message EntityProfileUpdateMsgProto { int64 newProfileIdLSB = 10; } -message EntityAddMsgProto { +message ProfileEntityMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; string entityType = 3; @@ -803,6 +803,8 @@ message EntityAddMsgProto { string entityProfileType = 6; int64 profileIdMSB = 7; int64 profileIdLSB = 8; + bool added = 9; + bool deleted = 10; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1538,8 +1540,7 @@ message ToCoreMsg { DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; - EntityAddMsgProto entityAddMsg = 55; - EntityDeleteMsg entityDeleteMsg = 56; + ProfileEntityMsgProto profileEntityMsg = 55; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ From 4fed2cd416b13b7f05159ba893f47dcd788fb13d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 9 Dec 2024 13:04:41 +0200 Subject: [PATCH 047/416] improved ability to fetch common arguments for profile cfs --- ...efaultCalculatedFieldExecutionService.java | 120 +++++++++--------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7ded80d013..f2629381b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -41,7 +41,6 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -89,7 +88,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -117,8 +116,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap states = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); - private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -196,7 +193,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); + fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> { + initializeStateForEntity(calculatedFieldCtx, assetId, commonArguments, callback); + }); + }); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -221,12 +222,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> { - if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { - updateStorage(tenantId, entityId, Optional.of(entry.getValue())); - } - return ArgumentEntry.createSingleValueArgument(entry.getValue()); - })); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); EntityId cfEntityId = calculatedField.getEntityId(); switch (cfEntityId.getEntityType()) { @@ -371,33 +367,71 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { - Map arguments = calculatedFieldCtx.getArguments(); + private void fetchCommonArguments(CalculatedFieldCtx calculatedFieldCtx, TbCallback callback, Consumer> onComplete) { Map argumentValues = new HashMap<>(); - AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedFieldCtx, entityId, argument), new FutureCallback<>() { + List> futures = new ArrayList<>(); + + calculatedFieldCtx.getArguments().forEach((key, argument) -> { + if (!EntityType.DEVICE_PROFILE.equals(argument.getEntityId().getEntityType()) && + !EntityType.ASSET_PROFILE.equals(argument.getEntityId().getEntityType())) { + futures.add(Futures.transform(fetchKvEntry(calculatedFieldCtx.getTenantId(), argument.getEntityId(), argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + } + }); + + Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override - public void onSuccess(ArgumentEntry result) { - argumentValues.put(key, result); - if (remaining.decrementAndGet() == 0) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - } + public void onSuccess(List results) { + onComplete.accept(argumentValues); + } + + @Override + public void onFailure(Throwable t) { + log.error("Failed to fetch common arguments", t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor); + } + + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { + initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); + } + + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map commonArguments, TbCallback callback) { + Map argumentValues = new HashMap<>(commonArguments); + List> futures = new ArrayList<>(); + + calculatedFieldCtx.getArguments().forEach((key, argument) -> { + if (!commonArguments.containsKey(key)) { + futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + } + }); + + Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { + @Override + public void onSuccess(List results) { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + callback.onSuccess(); } @Override public void onFailure(Throwable t) { - log.warn("Failed to initialize state for entity: [{}]", entityId, t); + log.error("Failed to initialize state for entity: [{}]", entityId, t); callback.onFailure(t); } - }, calculatedFieldCallbackExecutor)); + }, calculatedFieldCallbackExecutor); } private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); - if (EntityType.TENANT.equals(argumentEntityId.getEntityType()) || EntityType.CUSTOMER.equals(argumentEntityId.getEntityType())) { - return fetchFromStorage(tenantId, argumentEntityId, argument); - } EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; @@ -407,21 +441,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return fetchKvEntry(tenantId, entityId, argument); } - private ListenableFuture fetchFromStorage(TenantId tenantId, EntityId entityId, Argument argument) { - List kvEntries; - if (EntityType.TENANT.equals(entityId.getEntityType())) { - kvEntries = tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - } else { - kvEntries = customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - } - for (KvEntry kvEntry : kvEntries) { - if (kvEntry.getKey().equals(argument.getKey())) { - return Futures.immediateFuture(ArgumentEntry.createSingleValueArgument(kvEntry)); - } - } - return fetchKvEntry(tenantId, entityId, argument); - } - private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); @@ -450,26 +469,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> { - if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { - updateStorage(tenantId, entityId, kvEntry); - } - return ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)); - }, calculatedFieldExecutor); - } - - private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { - kvEntry.ifPresent(entry -> { - List kvEntries = switch (entityId.getEntityType()) { - case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - default -> null; - }; - if (kvEntries != null) { - kvEntries.removeIf(existingEntry -> existingEntry.getKey().equals(entry.getKey())); - kvEntries.add(entry); - } - }); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { From 86569c312e96e297ad1de55b31c681bf32c36a28 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 10 Dec 2024 12:21:13 +0200 Subject: [PATCH 048/416] added ability to perform calculations on the last records --- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../ctx/state/LastRecordsArgumentEntry.java | 2 +- .../LastRecordsCalculatedFieldState.java | 37 ++++++++++--------- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 10395a0d5d..b436e0421e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -51,7 +51,7 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { + if (!CalculatedFieldType.SIMPLE.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 93fabd3cb8..8b672d4de9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -37,7 +37,7 @@ public class LastRecordsArgumentEntry implements ArgumentEntry { @JsonIgnore @Override public Object getValue() { - return tsRecords.values(); + return tsRecords; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index dd69790236..fbcf72fda9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -17,13 +17,13 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; import java.util.Map; import java.util.TreeMap; @@ -56,23 +56,24 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - Map resultMap = new HashMap<>(); - arguments.forEach((key, argumentEntry) -> { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - long necessaryIntervalTs = calculateIntervalStart(System.currentTimeMillis(), argument.getTimeWindow()); - tsRecords.entrySet().removeIf(tsRecord -> calculateIntervalStart(tsRecord.getKey(), argument.getTimeWindow()) < necessaryIntervalTs); - resultMap.put(key, tsRecords); - }); - Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); - } - - private long calculateIntervalStart(long ts, long interval) { - return (ts / interval) * interval; + if (isValid(ctx.getArguments())) { + arguments.forEach((key, argumentEntry) -> { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + }); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); + } + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 99befa6e65..ba050d3b71 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -37,10 +37,10 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - Output output = ctx.getOutput(); if (isValid(ctx.getArguments())) { Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); return Futures.transform(resultFuture, result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), MoreExecutors.directExecutor() From a73affea23683c902a5ee45ad3b569c0463fd41b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 12 Dec 2024 17:04:24 +0200 Subject: [PATCH 049/416] removed CF type LAST_RECORDS and implemented this functionality as a script(added new argument type TS_ROLLING) --- ...efaultCalculatedFieldExecutionService.java | 49 ++++++------ .../service/cf/ctx/state/ArgumentEntry.java | 6 +- .../service/cf/ctx/state/ArgumentType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 26 +++--- .../cf/ctx/state/CalculatedFieldState.java | 2 - .../LastRecordsCalculatedFieldState.java | 79 ------------------- .../ctx/state/ScriptCalculatedFieldState.java | 14 +++- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- ...Entry.java => TsRollingArgumentEntry.java} | 4 +- .../common/data/cf/CalculatedFieldType.java | 2 +- .../BaseCalculatedFieldConfiguration.java | 4 +- .../CalculatedFieldConfiguration.java | 3 +- ...stRecordsCalculatedFieldConfiguration.java | 39 --------- .../dao/model/sql/CalculatedFieldEntity.java | 2 - ...efaultNativeCalculatedFieldRepository.java | 2 - 15 files changed, 64 insertions(+), 172 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{LastRecordsArgumentEntry.java => TsRollingArgumentEntry.java} (91%) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f2629381b2..050c2d7e73 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -73,12 +73,12 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -435,41 +435,41 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - if (CalculatedFieldType.LAST_RECORDS.equals(calculatedFieldCtx.getCfType())) { - return fetchLastRecords(tenantId, entityId, argument); - } return fetchKvEntry(tenantId, entityId, argument); } - private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getType()) { + case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); + case "ATTRIBUTE" -> transformSingleValueArgument( + Futures.transform( + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor) + ); + case "TS_LATEST" -> transformSingleValueArgument( + Futures.transform( + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor)); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); - ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, ArgumentEntry::createLastRecordsArgument, calculatedFieldExecutor); + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? ArgumentEntry.createTsRollingArgument(Collections.emptyList()) : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - ListenableFuture> kvEntryFuture = switch (argument.getType()) { - case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of( - new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) - )), - MoreExecutors.directExecutor()); - case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of( - new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) - )), - calculatedFieldExecutor); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); + private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldCallbackExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -547,7 +547,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return switch (calculatedFieldType) { case SIMPLE -> new SimpleCalculatedFieldState(); case SCRIPT -> new ScriptCalculatedFieldState(); - case LAST_RECORDS -> new LastRecordsCalculatedFieldState(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 6a62c0b1e3..f70d614123 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -32,7 +32,7 @@ import java.util.stream.Collectors; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), - @JsonSubTypes.Type(value = LastRecordsArgumentEntry.class, name = "LAST_RECORDS") + @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING") }) public interface ArgumentEntry { @@ -45,8 +45,8 @@ public interface ArgumentEntry { return new SingleValueArgumentEntry(kvEntry); } - static ArgumentEntry createLastRecordsArgument(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries.stream(). + static ArgumentEntry createTsRollingArgument(List kvEntries) { + return new TsRollingArgumentEntry(kvEntries.stream(). collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java index f2f0eac60d..360529a7e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentType { - SINGLE_VALUE, LAST_RECORDS + SINGLE_VALUE, TS_ROLLING } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 54df89e757..59b007a420 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -36,15 +35,22 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (arguments == null) { arguments = new HashMap<>(); } - arguments.putAll( - argumentValues.entrySet().stream() - .peek(entry -> { - if (entry.getValue() instanceof LastRecordsArgumentEntry) { - throw new IllegalArgumentException("Last records argument entry is not allowed for single calculated field state"); - } - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - ); + argumentValues.forEach((key, argumentEntry) -> { + ArgumentEntry existingArgumentEntry = arguments.get(key); + if (existingArgumentEntry != null) { + if (existingArgumentEntry instanceof SingleValueArgumentEntry) { + arguments.put(key, argumentEntry); + } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + } + } + } else { + arguments.put(key, argumentEntry); + } + }); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index f0882cbea4..a5ac6b2c47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -34,7 +33,6 @@ import java.util.Map; @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), - @JsonSubTypes.Type(value = LastRecordsCalculatedFieldState.class, name = "LAST_RECORDS") }) public interface CalculatedFieldState { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java deleted file mode 100644 index fbcf72fda9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; -import lombok.Data; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.service.cf.CalculatedFieldResult; - -import java.util.Map; -import java.util.TreeMap; - -@Data -public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { - - public LastRecordsCalculatedFieldState() { - } - - @Override - public CalculatedFieldType getType() { - return CalculatedFieldType.LAST_RECORDS; - } - - @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new TreeMap<>(); - } - argumentValues.forEach((key, argumentEntry) -> { - LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) - arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new TreeMap<>())); - if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { - existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - existingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); - } - }); - } - - @Override - public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - arguments.forEach((key, argumentEntry) -> { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); - }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); - Output output = ctx.getOutput(); - return Futures.transform(resultFuture, - result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), - MoreExecutors.directExecutor() - ); - } - return null; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index ba050d3b71..b9b98f9c5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -21,10 +21,12 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; +import java.util.TreeMap; @Data @Slf4j @@ -38,6 +40,16 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { if (isValid(ctx.getArguments())) { + arguments.forEach((key, argumentEntry) -> { + if (argumentEntry instanceof TsRollingArgumentEntry) { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + } + }); Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); @@ -46,7 +58,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { MoreExecutors.directExecutor() ); } - return null; + return Futures.immediateFuture(null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index fc97141806..491419b40a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -57,7 +57,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { Output output = ctx.getOutput(); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } - return null; + return Futures.immediateFuture(null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 8b672d4de9..1166da113d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -25,13 +25,13 @@ import java.util.TreeMap; @Data @NoArgsConstructor @AllArgsConstructor -public class LastRecordsArgumentEntry implements ArgumentEntry { +public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords; @Override public ArgumentType getType() { - return ArgumentType.LAST_RECORDS; + return ArgumentType.TS_ROLLING; } @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 63b6d8d1dd..89173b35b9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -17,6 +17,6 @@ package org.thingsboard.server.common.data.cf; public enum CalculatedFieldType { - SIMPLE, SCRIPT, LAST_RECORDS + SIMPLE, SCRIPT } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index f311fb737b..ac36991a61 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -69,10 +69,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel Argument argument = entry.getValue(); if (argument.getEntityId().equals(entityId)) { switch (argument.getType()) { - case "ATTRIBUTES": + case "ATTRIBUTE": linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); break; - case "TIME_SERIES": + case "TS_LATEST", "TS_ROLLING": linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); break; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 15f7a82c40..5c428bd628 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -35,8 +35,7 @@ import java.util.UUID; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), - @JsonSubTypes.Type(value = LastRecordsCalculatedFieldConfiguration.class, name = "LAST_RECORDS") + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") }) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java deleted file mode 100644 index c3f5804227..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration; - -import com.fasterxml.jackson.databind.JsonNode; -import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; - -import java.util.UUID; - -@Data -public class LastRecordsCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - - public LastRecordsCalculatedFieldConfiguration() { - } - - public LastRecordsCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - - @Override - public CalculatedFieldType getType() { - return CalculatedFieldType.LAST_RECORDS; - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index b06676f70b..6aaaf05836 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -124,7 +123,6 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return switch (CalculatedFieldType.valueOf(type)) { case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); }; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 2acd4d75c6..a5a2743f26 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,7 +29,6 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -140,7 +139,6 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF return switch (type) { case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); }; } From 4668bece03229c34aa74c335bd6602ddf9885c71 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 13 Dec 2024 11:14:56 +0200 Subject: [PATCH 050/416] fixed tests --- .../server/controller/CalculatedFieldControllerTest.java | 4 ++-- .../org/thingsboard/server/dao/service/AssetServiceTest.java | 4 ++-- .../server/dao/service/CalculatedFieldServiceTest.java | 4 ++-- .../thingsboard/server/dao/service/CustomerServiceTest.java | 4 ++-- .../org/thingsboard/server/dao/service/DeviceServiceTest.java | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 77ca268d12..ba1dfb1fec 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -140,7 +140,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -149,7 +149,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 87f2cb1f45..b0870f3dc5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -884,7 +884,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -893,7 +893,7 @@ public class AssetServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 77ed026b1d..9a1719e715 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -153,7 +153,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -162,7 +162,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 94c8440057..6671e0e821 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -379,7 +379,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -388,7 +388,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index d5394a3494..f2f8686bc3 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1222,7 +1222,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -1231,7 +1231,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); From 19f6f323260347e5af7eb0317dbe168dd9cb8e52 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 16 Dec 2024 12:47:55 +0200 Subject: [PATCH 051/416] moved logic when telemetry update to main cf service --- .../cf/CalculatedFieldExecutionService.java | 5 +- .../service/cf/CalculatedFieldResult.java | 6 +- ...efaultCalculatedFieldExecutionService.java | 149 +++++++++++++----- .../cf/ctx/CalculatedFieldEntityCtx.java | 5 +- .../service/cf/ctx/state/ArgumentEntry.java | 2 + .../ctx/state/BaseCalculatedFieldState.java | 22 ++- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../cf/ctx/state/CalculatedFieldState.java | 7 +- .../ctx/state/ScriptCalculatedFieldState.java | 35 ++-- .../ctx/state/SimpleCalculatedFieldState.java | 37 ++--- .../ctx/state/SingleValueArgumentEntry.java | 9 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 4 + .../DefaultTelemetrySubscriptionService.java | 78 +-------- 13 files changed, 186 insertions(+), 175 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 5a85529f6b..302e4b2511 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,20 +15,19 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; -import java.util.Map; +import java.util.List; public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 1f8a06c8fa..e8ea318bf6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -23,9 +23,9 @@ import java.util.Map; @Data public final class CalculatedFieldResult { - private String type; - private AttributeScope scope; - private Map resultMap; + private final String type; + private final AttributeScope scope; + private final Map resultMap; public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { this.type = type; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 050c2d7e73..c1a86ac36f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,16 +36,20 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -76,6 +80,8 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; import java.util.Collections; @@ -102,6 +108,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldService calculatedFieldService; private final AssetService assetService; private final DeviceService deviceService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; @@ -112,6 +120,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); @@ -130,6 +139,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(this::fetchCalculatedFields); + } + + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } @PreDestroy @@ -216,30 +235,77 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry) { try { - log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); - Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - - EntityId cfEntityId = calculatedField.getEntityId(); - switch (cfEntityId.getEntityType()) { - case ASSET_PROFILE, DEVICE_PROFILE -> { - boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); - if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); - } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - } + EntityType entityType = entityId.getEntityType(); + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { + EntityId profileId = null; + if (EntityType.ASSET.equals(entityType)) { + profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + } else if (EntityType.DEVICE.equals(entityType)) { + profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); + cfLinks.forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + Map attributes = link.getConfiguration().getAttributes(); + Map timeSeries = link.getConfiguration().getTimeSeries(); + Map updatedTelemetry = telemetry.stream() + .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .collect(Collectors.toMap( + entry -> getMappedKey(entry, attributes, timeSeries), + entry -> entry, + (v1, v2) -> v1 + )); + + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); + } + }); } - log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { - log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); + log.trace("Failed to update telemetry entityId: [{}]", entityId, e); + } + } + + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); + CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); + Map argumentValues = updatedTelemetry.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + + EntityId cfEntityId = calculatedField.getEntityId(); + switch (cfEntityId.getEntityType()) { + case ASSET_PROFILE, DEVICE_PROFILE -> { + boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); + if (isCommonEntity) { + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + } else { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + } + } + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + } + log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); + } + + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); } + return entry.getKey(); } @Override @@ -493,26 +559,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (state == null) { state = createStateByType(calculatedFieldCtx.getCfType()); } - state.initState(argumentValues); - calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); - } - } + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + + boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); + if (allArgsPresent) { + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + } + } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); + } + }, MoreExecutors.directExecutor()); } - }, MoreExecutors.directExecutor()); - + } } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { @@ -520,14 +589,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (stateStr == null) { return new CalculatedFieldEntityCtx(entityCtxId, null); } - return JacksonUtil.fromString(rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)), CalculatedFieldEntityCtx.class); + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + TbMsgType msgType = "ATTRIBUTE".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTE".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java index 7a8384b6bf..e6dc021951 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java @@ -16,17 +16,16 @@ package org.thingsboard.server.service.cf.ctx; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data +@NoArgsConstructor public class CalculatedFieldEntityCtx { private CalculatedFieldEntityCtxId id; private CalculatedFieldState state; - public CalculatedFieldEntityCtx() { - } - public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { this.id = id; this.state = state; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index f70d614123..78222244c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -41,6 +41,8 @@ public interface ArgumentEntry { Object getValue(); + boolean hasUpdatedValue(ArgumentEntry entry); + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { return new SingleValueArgumentEntry(kvEntry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 59b007a420..ae6fc9033a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -31,26 +32,39 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public boolean updateState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } + AtomicBoolean stateUpdated = new AtomicBoolean(false); argumentValues.forEach((key, argumentEntry) -> { ArgumentEntry existingArgumentEntry = arguments.get(key); if (existingArgumentEntry != null) { if (existingArgumentEntry instanceof SingleValueArgumentEntry) { - arguments.put(key, argumentEntry); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + arguments.put(key, argumentEntry); + stateUpdated.set(true); + } } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + stateUpdated.set(true); + } } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + stateUpdated.set(true); + } + } } } else { arguments.put(key, argumentEntry); + stateUpdated.set(true); } }); + return stateUpdated.get(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index b436e0421e..2cd5c68144 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -35,7 +35,7 @@ public class CalculatedFieldCtx { private TenantId tenantId; private EntityId entityId; private CalculatedFieldType cfType; - private Map arguments; + private final Map arguments; private Output output; private String expression; private TbelInvokeService tbelInvokeService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index a5ac6b2c47..3c4a680df1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -41,11 +40,7 @@ public interface CalculatedFieldState { Map getArguments(); - default boolean isValid(Map arguments) { - return getArguments().keySet().containsAll(arguments.keySet()); - } - - void initState(Map argumentValues); + boolean updateState(Map argumentValues); ListenableFuture performCalculation(CalculatedFieldCtx ctx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index b9b98f9c5e..87429050de 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -39,26 +39,23 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - arguments.forEach((key, argumentEntry) -> { - if (argumentEntry instanceof TsRollingArgumentEntry) { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + arguments.forEach((key, argumentEntry) -> { + if (argumentEntry instanceof TsRollingArgumentEntry) { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); } - }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); - Output output = ctx.getOutput(); - return Futures.transform(resultFuture, - result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), - MoreExecutors.directExecutor() - ); - } - return Futures.immediateFuture(null); + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + } + }); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 491419b40a..e16d310b3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -37,27 +37,24 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - String expression = ctx.getExpression(); - ThreadLocal customExpression = new ThreadLocal<>(); - var expr = customExpression.get(); - if (expr == null) { - expr = new ExpressionBuilder(expression) - .implicitMultiplication(true) - .variables(this.arguments.keySet()) - .build(); - customExpression.set(expr); - } - Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); - expr.setVariables(variables); - - double expressionResult = expr.evaluate(); - - Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); + String expression = ctx.getExpression(); + ThreadLocal customExpression = new ThreadLocal<>(); + var expr = customExpression.get(); + if (expr == null) { + expr = new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(this.arguments.keySet()) + .build(); + customExpression.set(expr); } - return Futures.immediateFuture(null); + Map variables = new HashMap<>(); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); + expr.setVariables(variables); + + double expressionResult = expr.evaluate(); + + Output output = ctx.getOutput(); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e0db8c50fb..e6cb24b970 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -16,19 +16,18 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @Data +@NoArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - public SingleValueArgumentEntry() { - } - public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry) { this.ts = ((TsKvEntry) entry).getTs(); @@ -48,4 +47,8 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return value; } + @Override + public boolean hasUpdatedValue(ArgumentEntry entry) { + return this.ts != ((SingleValueArgumentEntry) entry).getTs(); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1166da113d..104d0ae90c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -40,4 +40,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return tsRecords; } + @Override + public boolean hasUpdatedValue(ArgumentEntry entry) { + return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ad64204e0f..b94b319e71 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -32,11 +32,7 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -44,7 +40,6 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -52,14 +47,11 @@ import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import java.util.ArrayList; @@ -73,7 +65,6 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -87,11 +78,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; - private final CalculatedFieldService calculatedFieldService; private final CalculatedFieldExecutionService calculatedFieldExecutionService; - private final TbAssetProfileCache assetProfileCache; - private final TbDeviceProfileCache deviceProfileCache; - private ExecutorService tsCallBackExecutor; @Value("${sql.ts.value_no_xss_validation:false}") @@ -102,19 +89,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, - CalculatedFieldService calculatedFieldService, - CalculatedFieldExecutionService calculatedFieldExecutionService, - TbAssetProfileCache assetProfileCache, - TbDeviceProfileCache deviceProfileCache) { + CalculatedFieldExecutionService calculatedFieldExecutionService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; - this.calculatedFieldService = calculatedFieldService; this.calculatedFieldExecutionService = calculatedFieldExecutionService; - this.assetProfileCache = assetProfileCache; - this.deviceProfileCache = deviceProfileCache; } @PostConstruct @@ -201,7 +182,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addEntityViewCallback(tenantId, entityId, ts); - updateTelemetryInCalculatedFields(tenantId, entityId, ts); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, ts); } private void saveWithoutLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { @@ -210,55 +191,6 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } - private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { - EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { - EntityId profileId = null; - if (EntityType.ASSET.equals(entityType)) { - profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else if (EntityType.DEVICE.equals(entityType)) { - profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - } - List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); - if (!cfLinks.isEmpty()) { - cfLinks.forEach(link -> { - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map attributes = link.getConfiguration().getAttributes(); - Map timeSeries = link.getConfiguration().getTimeSeries(); - Map updatedTelemetry = telemetry.stream() - .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, attributes, timeSeries), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); - } - }); - } - } - } - - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - } - private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), @@ -335,7 +267,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); - updateTelemetryInCalculatedFields(tenantId, entityId, attributes); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, attributes); } @Override @@ -343,7 +275,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope.name(), attributes, notifyDevice)); - updateTelemetryInCalculatedFields(tenantId, entityId, attributes); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, attributes); } @Override @@ -357,7 +289,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); - updateTelemetryInCalculatedFields(tenantId, entityId, ts); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, ts); } @Override From 481753b8f0e0b15f3174f65ee18fca33eb0a7568 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 17 Dec 2024 13:41:23 +0200 Subject: [PATCH 052/416] added restriction for number of cfs/arguments/ts rolling values --- .../service/cf/ctx/state/ArgumentEntry.java | 3 ++ .../ctx/state/BaseCalculatedFieldState.java | 9 ++--- .../ctx/state/SingleValueArgumentEntry.java | 8 ++++ .../cf/ctx/state/TsRollingArgumentEntry.java | 37 +++++++++++++++++-- .../cf/DefaultTbCalculatedFieldService.java | 18 +++++++++ 5 files changed, 67 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 78222244c9..ba7e094f77 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -52,4 +52,7 @@ public interface ArgumentEntry { collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } + @JsonIgnore + ArgumentEntry copy(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index ae6fc9033a..b73ac51798 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -42,25 +42,24 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (existingArgumentEntry != null) { if (existingArgumentEntry instanceof SingleValueArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - arguments.put(key, argumentEntry); + arguments.put(key, argumentEntry.copy()); stateUpdated.set(true); } } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + existingTsRollingArgumentEntry.addAllTsRecords(tsRollingArgumentEntry.getTsRecords()); stateUpdated.set(true); } } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + existingTsRollingArgumentEntry.addTsRecord(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); stateUpdated.set(true); } - } } } else { - arguments.put(key, argumentEntry); + arguments.put(key, argumentEntry.copy()); stateUpdated.set(true); } }); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e6cb24b970..81a57580db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -23,6 +24,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; @Data @NoArgsConstructor +@AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; @@ -51,4 +53,10 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public boolean hasUpdatedValue(ArgumentEntry entry) { return this.ts != ((SingleValueArgumentEntry) entry).getTs(); } + + @Override + public ArgumentEntry copy() { + return new SingleValueArgumentEntry(this.ts, this.value); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 104d0ae90c..49aae15ac1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -16,18 +16,26 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; +import java.util.Map; import java.util.TreeMap; @Data @NoArgsConstructor -@AllArgsConstructor +@Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { - private TreeMap tsRecords; + private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; + + private TreeMap tsRecords = new TreeMap<>(); + + public TsRollingArgumentEntry(TreeMap tsRecords) { + addAllTsRecords(tsRecords); + } @Override public ArgumentType getType() { @@ -44,4 +52,27 @@ public class TsRollingArgumentEntry implements ArgumentEntry { public boolean hasUpdatedValue(ArgumentEntry entry) { return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); } + + @Override + public ArgumentEntry copy() { + return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); + } + + public void addTsRecord(Long key, Object value) { + if (NumberUtils.isParsable(value.toString())) { + tsRecords.put(key, value); + if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { + tsRecords.pollFirstEntry(); + } + } else { + log.warn("Argument type 'TS_ROLLING' only supports numeric values."); + } + } + + public void addAllTsRecords(Map newRecords) { + for (Map.Entry entry : newRecords.entrySet()) { + addTsRecord(entry.getKey(), entry.getValue()); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 4d28ff55ac..2e6e975636 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -46,6 +46,9 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { + private static final int MAX_ARGUMENT_SIZE = 10; + private static final int MAX_CALCULATED_FIELD_NUMBER = 10; + private final CalculatedFieldService calculatedFieldService; @Override @@ -53,7 +56,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = calculatedField.getTenantId(); try { + checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId()); + checkArgumentSize(calculatedField.getConfiguration()); checkReferencedEntities(calculatedField.getConfiguration(), user); CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); @@ -105,6 +110,19 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } + private void checkArgumentSize(CalculatedFieldConfiguration calculatedFieldConfig) { + if (calculatedFieldConfig.getArguments().size() > MAX_ARGUMENT_SIZE) { + throw new IllegalArgumentException("Too many arguments: " + calculatedFieldConfig.getArguments().size() + ". Max number of argument is " + MAX_ARGUMENT_SIZE); + } + } + + private void checkCalculatedFieldNumber(TenantId tenantId, EntityId entityId) { + int numberOfCalculatedFieldsByEntityId = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, entityId).size(); + if (numberOfCalculatedFieldsByEntityId >= MAX_CALCULATED_FIELD_NUMBER) { + throw new IllegalArgumentException("Max number of calculated fields for entity is " + MAX_CALCULATED_FIELD_NUMBER); + } + } + private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); From f929de42097c4929cae2d8970ea45dfced2b12b0 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 18 Dec 2024 13:12:39 +0200 Subject: [PATCH 053/416] onAddedPartitions() impl --- ...efaultCalculatedFieldExecutionService.java | 65 +++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c1a86ac36f..7e5ea5b5ec 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -63,6 +64,7 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.asset.AssetService; @@ -173,13 +175,66 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override protected Map>> onAddedPartitions(Set addedPartitions) { - // TODO: implementation for cluster mode - return Map.of(); + var result = new HashMap>>(); + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + Map> tpiCalculatedFieldMap = new HashMap<>(); + + for (CalculatedField cf : cfs) { + TopicPartitionInfo tpi; + try { + tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); + } catch (Exception e) { + log.warn("Failed to resolve partition for CalculatedField [{}], tenant [{}]. Reason: {}", + cf.getId(), cf.getTenantId(), e.getMessage()); + continue; + } + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { + tpiCalculatedFieldMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(cf); + } + } + + for (var entry : tpiCalculatedFieldMap.entrySet()) { + for (List partition : Lists.partition(entry.getValue(), 1000)) { + log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); + var future = calculatedFieldExecutor.submit(() -> { + try { + for (CalculatedField cf : partition) { + if (EntityType.ASSET_PROFILE.equals(cf.getEntityId().getEntityType()) || EntityType.DEVICE_PROFILE.equals(cf.getEntityId().getEntityType())) { + getOrFetchFromDBProfileEntities(cf.getTenantId(), cf.getEntityId()) + .forEach(entityId -> restoreState(cf, entityId)); + } else { + restoreState(cf, cf.getEntityId()); + } + } + } catch (Throwable t) { + log.error("Unexpected exception while restoring CalculatedField states", t); + throw t; + } + }); + result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); + } + } + return result; + } + + private void restoreState(CalculatedField cf, EntityId entityId) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId()); + String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + + if (storedState != null) { + CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); + calculatedFieldsCtx.putIfAbsent(cf.getId(), new CalculatedFieldCtx(cf, tbelInvokeService)); + states.put(ctxId, restoredCtx); + log.info("Restored state for CalculatedField [{}]", cf.getId()); + } else { + log.warn("No state found for CalculatedField [{}], entity [{}].", cf.getId(), entityId); + } } @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - // TODO: implementation for cluster mode + calculatedFields.remove(entityId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(entityId.getId())); } @Override @@ -213,8 +268,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> { - initializeStateForEntity(calculatedFieldCtx, assetId, commonArguments, callback); + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { + initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArguments, callback); }); }); } From 3ccf0b0d3d1599f57cb41d2013cdbabd763b431c Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 18 Dec 2024 15:38:20 +0200 Subject: [PATCH 054/416] Refactor edge: remove deprecated proto usage --- .../service/edge/EdgeContextComponent.java | 138 +++-- .../service/edge/EdgeMsgConstructorUtils.java | 544 ++++++++++++++++++ .../service/edge/rpc/EdgeGrpcService.java | 2 +- .../service/edge/rpc/EdgeGrpcSession.java | 103 +--- .../BaseMsgConstructorFactory.java | 40 -- .../edge/rpc/constructor/MsgConstructor.java | 20 - .../alarm/AlarmMsgConstructor.java | 30 - .../alarm/AlarmMsgConstructorFactory.java | 26 - .../alarm/AlarmMsgConstructorV1.java | 50 -- .../alarm/AlarmMsgConstructorV2.java | 36 -- .../asset/AssetMsgConstructor.java | 36 -- .../asset/AssetMsgConstructorFactory.java | 26 - .../asset/AssetMsgConstructorV1.java | 94 --- .../asset/AssetMsgConstructorV2.java | 44 -- .../asset/BaseAssetMsgConstructor.java | 41 -- .../customer/BaseCustomerMsgConstructor.java | 31 - .../customer/CustomerMsgConstructor.java | 29 - .../CustomerMsgConstructorFactory.java | 26 - .../customer/CustomerMsgConstructorV1.java | 65 --- .../customer/CustomerMsgConstructorV2.java | 35 -- .../BaseDashboardMsgConstructor.java | 32 -- .../dashboard/DashboardMsgConstructor.java | 30 - .../DashboardMsgConstructorFactory.java | 26 - .../dashboard/DashboardMsgConstructorV1.java | 57 -- .../dashboard/DashboardMsgConstructorV2.java | 36 -- .../device/BaseDeviceMsgConstructor.java | 102 ---- .../device/DeviceMsgConstructor.java | 47 -- .../device/DeviceMsgConstructorFactory.java | 26 - .../device/DeviceMsgConstructorV1.java | 141 ----- .../device/DeviceMsgConstructorV2.java | 52 -- .../constructor/edge/EdgeMsgConstructor.java | 47 -- .../BaseEntityViewMsgConstructor.java | 32 -- .../entityview/EntityViewMsgConstructor.java | 30 - .../EntityViewMsgConstructorFactory.java | 26 - .../EntityViewMsgConstructorV1.java | 61 -- .../EntityViewMsgConstructorV2.java | 36 -- .../NotificationMsgConstructor.java | 43 -- .../NotificationMsgConstructorImpl.java | 75 --- .../oauth2/OAuth2MsgConstructor.java | 60 -- .../ota/BaseOtaPackageMsgConstructor.java | 31 - .../ota/OtaPackageMsgConstructor.java | 29 - .../ota/OtaPackageMsgConstructorFactory.java | 26 - .../ota/OtaPackageMsgConstructorV1.java | 72 --- .../ota/OtaPackageMsgConstructorV2.java | 35 -- .../queue/BaseQueueMsgConstructor.java | 31 - .../queue/QueueMsgConstructor.java | 29 - .../queue/QueueMsgConstructorFactory.java | 26 - .../queue/QueueMsgConstructorV1.java | 74 --- .../queue/QueueMsgConstructorV2.java | 35 -- .../relation/RelationMsgConstructor.java | 26 - .../RelationMsgConstructorFactory.java | 26 - .../relation/RelationMsgConstructorV1.java | 48 -- .../relation/RelationMsgConstructorV2.java | 34 -- .../resource/BaseResourceMsgConstructor.java | 31 - .../resource/ResourceMsgConstructor.java | 29 - .../ResourceMsgConstructorFactory.java | 26 - .../resource/ResourceMsgConstructorV1.java | 56 -- .../resource/ResourceMsgConstructorV2.java | 35 -- .../BaseRuleChainMetadataConstructor.java | 133 ----- .../rule/BaseRuleChainMsgConstructor.java | 44 -- .../rule/RuleChainMetadataConstructor.java | 30 - .../RuleChainMetadataConstructorFactory.java | 36 -- .../RuleChainMetadataConstructorV330.java | 166 ------ .../RuleChainMetadataConstructorV340.java | 43 -- .../RuleChainMetadataConstructorV362.java | 29 - .../rule/RuleChainMsgConstructor.java | 38 -- .../rule/RuleChainMsgConstructorFactory.java | 26 - .../rule/RuleChainMsgConstructorV1.java | 45 -- .../rule/RuleChainMsgConstructorV2.java | 39 -- .../settings/AdminSettingsMsgConstructor.java | 25 - .../AdminSettingsMsgConstructorFactory.java | 26 - .../AdminSettingsMsgConstructorV1.java | 39 -- .../AdminSettingsMsgConstructorV2.java | 32 -- .../telemetry/EntityDataMsgConstructor.java | 110 ---- .../tenant/TenantMsgConstructor.java | 31 - .../tenant/TenantMsgConstructorFactory.java | 26 - .../tenant/TenantMsgConstructorV1.java | 111 ---- .../tenant/TenantMsgConstructorV2.java | 42 -- .../user/BaseUserMsgConstructor.java | 31 - .../constructor/user/UserMsgConstructor.java | 33 -- .../user/UserMsgConstructorFactory.java | 26 - .../user/UserMsgConstructorV1.java | 63 -- .../user/UserMsgConstructorV2.java | 42 -- .../widget/BaseWidgetMsgConstructor.java | 43 -- .../widget/WidgetMsgConstructor.java | 39 -- .../widget/WidgetMsgConstructorFactory.java | 26 - .../widget/WidgetMsgConstructorV1.java | 115 ---- .../widget/WidgetMsgConstructorV2.java | 48 -- .../edge/rpc/processor/BaseEdgeProcessor.java | 3 +- .../processor/BaseEdgeProcessorFactory.java | 40 -- .../edge/rpc/processor/EdgeProcessor.java | 13 + .../processor/alarm/AlarmEdgeProcessor.java | 105 +--- .../alarm/AlarmEdgeProcessorFactory.java | 26 - .../processor/alarm/AlarmEdgeProcessorV1.java | 69 --- .../processor/alarm/AlarmEdgeProcessorV2.java | 44 -- .../rpc/processor/alarm/AlarmProcessor.java | 10 - .../processor/alarm/BaseAlarmProcessor.java | 29 +- .../comment/AlarmCommentEdgeProcessor.java | 93 +++ .../alarm/comment/AlarmCommentProcessor.java} | 17 +- .../processor/asset/AssetEdgeProcessor.java | 25 +- .../asset/AssetEdgeProcessorFactory.java | 26 - .../processor/asset/AssetEdgeProcessorV1.java | 60 -- .../processor/asset/AssetEdgeProcessorV2.java | 44 -- .../rpc/processor/asset/AssetProcessor.java | 5 - .../processor/asset/BaseAssetProcessor.java | 5 +- .../profile/AssetProfileEdgeProcessor.java | 36 +- .../AssetProfileEdgeProcessorFactory.java | 26 - .../profile/AssetProfileEdgeProcessorV1.java | 65 --- .../profile/AssetProfileEdgeProcessorV2.java | 57 -- .../asset/profile/AssetProfileProcessor.java | 5 - .../profile/BaseAssetProfileProcessor.java | 5 +- .../customer/CustomerEdgeProcessor.java | 21 +- .../dashboard/BaseDashboardProcessor.java | 5 +- .../dashboard/DashboardEdgeProcessor.java | 21 +- .../DashboardEdgeProcessorFactory.java | 26 - .../dashboard/DashboardEdgeProcessorV1.java | 54 -- .../dashboard/DashboardEdgeProcessorV2.java | 37 -- .../dashboard/DashboardProcessor.java | 4 - .../processor/device/BaseDeviceProcessor.java | 9 +- .../processor/device/DeviceEdgeProcessor.java | 31 +- .../device/DeviceEdgeProcessorFactory.java | 26 - .../device/DeviceEdgeProcessorV1.java | 79 --- .../device/DeviceEdgeProcessorV2.java | 50 -- .../rpc/processor/device/DeviceProcessor.java | 5 - .../profile/BaseDeviceProfileProcessor.java | 5 +- .../profile/DeviceProfileEdgeProcessor.java | 50 +- .../DeviceProfileEdgeProcessorFactory.java | 26 - .../profile/DeviceProfileEdgeProcessorV1.java | 89 --- .../profile/DeviceProfileEdgeProcessorV2.java | 57 -- .../profile/DeviceProfileProcessor.java | 5 - ...rocessor.java => EdgeEntityProcessor.java} | 41 +- .../entityview/BaseEntityViewProcessor.java | 5 +- .../entityview/EntityViewEdgeProcessor.java | 29 +- .../entityview/EntityViewProcessor.java | 5 - .../EntityViewProcessorFactory.java | 26 - .../entityview/EntityViewProcessorV1.java | 66 --- .../entityview/EntityViewProcessorV2.java | 44 -- .../NotificationEdgeProcessor.java | 136 ----- .../NotificationRuleEdgeProcessor.java | 68 +++ .../NotificationTargetEdgeProcessor.java | 68 +++ .../NotificationTemplateEdgeProcessor.java | 68 +++ ...rocessor.java => DomainEdgeProcessor.java} | 56 +- .../oauth2/OAuth2ClientEdgeProcessor.java | 67 +++ .../ota/OtaPackageEdgeProcessor.java | 18 +- .../processor/queue/QueueEdgeProcessor.java | 16 +- .../relation/BaseRelationProcessor.java | 5 +- .../relation/RelationEdgeProcessor.java | 38 +- .../RelationEdgeProcessorFactory.java | 26 - .../relation/RelationEdgeProcessorV1.java | 53 -- .../relation/RelationEdgeProcessorV2.java | 35 -- .../processor/relation/RelationProcessor.java | 5 - .../resource/BaseResourceProcessor.java | 5 +- .../resource/ResourceEdgeProcessor.java | 25 +- .../ResourceEdgeProcessorFactory.java | 26 - .../resource/ResourceEdgeProcessorV1.java | 49 -- .../resource/ResourceEdgeProcessorV2.java | 37 -- .../processor/resource/ResourceProcessor.java | 5 - .../rule/RuleChainEdgeProcessor.java | 43 +- .../rule/RuleChainMetadataEdgeProcessor.java | 54 ++ .../settings/AdminSettingsEdgeProcessor.java | 15 +- .../telemetry/BaseTelemetryProcessor.java | 8 +- .../processor/tenant/TenantEdgeProcessor.java | 16 +- .../tenant/TenantProfileEdgeProcessor.java | 14 +- .../rpc/processor/user/UserEdgeProcessor.java | 20 +- .../widget/WidgetBundleEdgeProcessor.java | 16 +- .../widget/WidgetTypeEdgeProcessor.java | 16 +- .../queue/DefaultTbEdgeConsumerService.java | 31 +- .../thingsboard/edge/rpc/EdgeGrpcClient.java | 2 +- common/edge-api/src/main/proto/edge.proto | 262 +-------- 169 files changed, 1396 insertions(+), 6142 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/BaseMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/MsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/BaseAssetMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/BaseCustomerMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/BaseDashboardMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/BaseDeviceMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/edge/EdgeMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/BaseEntityViewMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/notification/NotificationMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/notification/NotificationMsgConstructorImpl.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/oauth2/OAuth2MsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/BaseOtaPackageMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/BaseQueueMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/BaseResourceMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/BaseRuleChainMetadataConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/BaseRuleChainMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV330.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV340.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV362.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/telemetry/EntityDataMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/BaseUserMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/BaseWidgetMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV2.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentEdgeProcessor.java rename application/src/main/java/org/thingsboard/server/service/edge/rpc/{constructor/alarm/BaseAlarmMsgConstructor.java => processor/alarm/comment/AlarmCommentProcessor.java} (54%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorV2.java rename application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/{EdgeProcessor.java => EdgeEntityProcessor.java} (93%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationEdgeProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationRuleEdgeProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTargetEdgeProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTemplateEdgeProcessor.java rename application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/{OAuth2EdgeProcessor.java => DomainEdgeProcessor.java} (53%) create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2ClientEdgeProcessor.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorV2.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorFactory.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV1.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV2.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainMetadataEdgeProcessor.java diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index ee50bbb4a3..ce0725a627 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -17,11 +17,11 @@ package org.thingsboard.server.service.edge; import lombok.Data; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.alarm.AlarmService; @@ -55,34 +55,28 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.EdgeEventStorageSettings; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; -import org.thingsboard.server.service.edge.rpc.constructor.asset.AssetMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.device.DeviceMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.edge.EdgeMsgConstructor; -import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmEdgeProcessorFactory; +import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; +import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmProcessor; +import org.thingsboard.server.service.edge.rpc.processor.alarm.comment.AlarmCommentProcessor; import org.thingsboard.server.service.edge.rpc.processor.asset.AssetEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.asset.AssetEdgeProcessorFactory; import org.thingsboard.server.service.edge.rpc.processor.asset.profile.AssetProfileEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.asset.profile.AssetProfileEdgeProcessorFactory; import org.thingsboard.server.service.edge.rpc.processor.customer.CustomerEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.dashboard.DashboardEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.dashboard.DashboardEdgeProcessorFactory; import org.thingsboard.server.service.edge.rpc.processor.device.DeviceEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.device.DeviceEdgeProcessorFactory; import org.thingsboard.server.service.edge.rpc.processor.device.profile.DeviceProfileEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.device.profile.DeviceProfileEdgeProcessorFactory; -import org.thingsboard.server.service.edge.rpc.processor.edge.EdgeProcessor; +import org.thingsboard.server.service.edge.rpc.processor.edge.EdgeEntityProcessor; import org.thingsboard.server.service.edge.rpc.processor.entityview.EntityViewEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.entityview.EntityViewProcessorFactory; -import org.thingsboard.server.service.edge.rpc.processor.notification.NotificationEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.oauth2.OAuth2EdgeProcessor; +import org.thingsboard.server.service.edge.rpc.processor.notification.NotificationRuleEdgeProcessor; +import org.thingsboard.server.service.edge.rpc.processor.notification.NotificationTargetEdgeProcessor; +import org.thingsboard.server.service.edge.rpc.processor.notification.NotificationTemplateEdgeProcessor; +import org.thingsboard.server.service.edge.rpc.processor.oauth2.DomainEdgeProcessor; +import org.thingsboard.server.service.edge.rpc.processor.oauth2.OAuth2ClientEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.ota.OtaPackageEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.queue.QueueEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.relation.RelationEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.relation.RelationEdgeProcessorFactory; import org.thingsboard.server.service.edge.rpc.processor.resource.ResourceEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.resource.ResourceEdgeProcessorFactory; import org.thingsboard.server.service.edge.rpc.processor.rule.RuleChainEdgeProcessor; +import org.thingsboard.server.service.edge.rpc.processor.rule.RuleChainMetadataEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.settings.AdminSettingsEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.telemetry.TelemetryEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.tenant.TenantEdgeProcessor; @@ -93,12 +87,51 @@ import org.thingsboard.server.service.edge.rpc.processor.widget.WidgetTypeEdgePr import org.thingsboard.server.service.edge.rpc.sync.EdgeRequestsService; import org.thingsboard.server.service.executors.GrpcCallbackExecutorService; +import javax.annotation.PostConstruct; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + @Lazy @Data @Component @TbCoreComponent public class EdgeContextComponent { + private Map processorMap; + + @PostConstruct + private void initProcessorMap() { + Map map = new HashMap<>(); + map.put(EdgeEventType.ADMIN_SETTINGS, adminSettingsProcessor); + map.put(EdgeEventType.ALARM, alarmProcessor); + map.put(EdgeEventType.ALARM_COMMENT, alarmCommentProcessor); + map.put(EdgeEventType.ASSET, assetProcessor); + map.put(EdgeEventType.ASSET_PROFILE, assetProfileProcessor); + map.put(EdgeEventType.CUSTOMER, customerProcessor); + map.put(EdgeEventType.DASHBOARD, dashboardProcessor); + map.put(EdgeEventType.DEVICE, deviceProcessor); + map.put(EdgeEventType.DEVICE_PROFILE, deviceProfileProcessor); + map.put(EdgeEventType.DOMAIN, domainProcessor); + map.put(EdgeEventType.EDGE, edgeEntityProcessor); + map.put(EdgeEventType.ENTITY_VIEW, entityViewProcessor); + map.put(EdgeEventType.NOTIFICATION_RULE, notificationRuleProcessor); + map.put(EdgeEventType.NOTIFICATION_TARGET, notificationTargetProcessor); + map.put(EdgeEventType.NOTIFICATION_TEMPLATE, notificationTemplateProcessor); + map.put(EdgeEventType.OTA_PACKAGE, otaPackageProcessor); + map.put(EdgeEventType.QUEUE, queueProcessor); + map.put(EdgeEventType.RELATION, relationProcessor); + map.put(EdgeEventType.RULE_CHAIN, ruleChainProcessor); + map.put(EdgeEventType.RULE_CHAIN_METADATA, ruleChainMetadataProcessor); + map.put(EdgeEventType.TB_RESOURCE, resourceProcessor); + map.put(EdgeEventType.TENANT, tenantProcessor); + map.put(EdgeEventType.TENANT_PROFILE, tenantProfileProcessor); + map.put(EdgeEventType.USER, userProcessor); + map.put(EdgeEventType.WIDGETS_BUNDLE, widgetBundleProcessor); + map.put(EdgeEventType.WIDGET_TYPE, widgetTypeProcessor); + this.processorMap = Collections.unmodifiableMap(map); + } + // services @Autowired private AdminSettingsService adminSettingsService; @@ -205,7 +238,10 @@ public class EdgeContextComponent { private AdminSettingsEdgeProcessor adminSettingsProcessor; @Autowired - private AlarmEdgeProcessor alarmProcessor; + private AlarmProcessor alarmProcessor; + + @Autowired + private AlarmCommentProcessor alarmCommentProcessor; @Autowired private AssetEdgeProcessor assetProcessor; @@ -226,19 +262,28 @@ public class EdgeContextComponent { private DeviceProfileEdgeProcessor deviceProfileProcessor; @Autowired - private EdgeProcessor edgeProcessor; + private EdgeEntityProcessor edgeEntityProcessor; @Autowired private EntityViewEdgeProcessor entityViewProcessor; @Autowired - private NotificationEdgeProcessor notificationEdgeProcessor; + private NotificationRuleProcessor ruleProcessor; + + @Autowired + private NotificationRuleEdgeProcessor notificationRuleProcessor; + + @Autowired + private NotificationTargetEdgeProcessor notificationTargetProcessor; + + @Autowired + private NotificationTemplateEdgeProcessor notificationTemplateProcessor; @Autowired - private NotificationRuleProcessor notificationRuleProcessor; + private DomainEdgeProcessor domainProcessor; @Autowired - private OAuth2EdgeProcessor oAuth2EdgeProcessor; + private OAuth2ClientEdgeProcessor oAuth2ClientProcessor; @Autowired private OtaPackageEdgeProcessor otaPackageProcessor; @@ -255,6 +300,9 @@ public class EdgeContextComponent { @Autowired private RuleChainEdgeProcessor ruleChainProcessor; + @Autowired + private RuleChainMetadataEdgeProcessor ruleChainMetadataProcessor; + @Autowired private TelemetryEdgeProcessor telemetryProcessor; @@ -273,44 +321,6 @@ public class EdgeContextComponent { @Autowired private WidgetTypeEdgeProcessor widgetTypeProcessor; - // msg constructors - @Autowired - private EdgeMsgConstructor edgeMsgConstructor; - - // factories - @Autowired - private AlarmEdgeProcessorFactory alarmEdgeProcessorFactory; - - @Autowired - private AssetEdgeProcessorFactory assetEdgeProcessorFactory; - - @Autowired - private AssetMsgConstructorFactory assetMsgConstructorFactory; - - @Autowired - private AssetProfileEdgeProcessorFactory assetProfileEdgeProcessorFactory; - - @Autowired - private DashboardEdgeProcessorFactory dashboardEdgeProcessorFactory; - - @Autowired - private DeviceEdgeProcessorFactory deviceEdgeProcessorFactory; - - @Autowired - private DeviceMsgConstructorFactory deviceMsgConstructorFactory; - - @Autowired - private DeviceProfileEdgeProcessorFactory deviceProfileEdgeProcessorFactory; - - @Autowired - private EntityViewProcessorFactory entityViewProcessorFactory; - - @Autowired - private RelationEdgeProcessorFactory relationEdgeProcessorFactory; - - @Autowired - private ResourceEdgeProcessorFactory resourceEdgeProcessorFactory; - // config @Autowired private EdgeEventStorageSettings edgeEventStorageSettings; @@ -319,4 +329,12 @@ public class EdgeContextComponent { @Autowired private GrpcCallbackExecutorService grpcCallbackExecutorService; + public EdgeProcessor getProcessor(EdgeEventType edgeEventType) { + EdgeProcessor processor = processorMap.get(edgeEventType); + if (processor == null) { + throw new UnsupportedOperationException("No processor found for EdgeEventType: " + edgeEventType); + } + return processor; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java new file mode 100644 index 0000000000..6e0419ee5f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java @@ -0,0 +1,544 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import com.google.gson.reflect.TypeToken; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.adaptor.JsonConverter; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmComment; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.domain.DomainInfo; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.DomainId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.NotificationRuleId; +import org.thingsboard.server.common.data.id.NotificationTargetId; +import org.thingsboard.server.common.data.id.NotificationTemplateId; +import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TbResourceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.notification.rule.NotificationRule; +import org.thingsboard.server.common.data.notification.targets.NotificationTarget; +import org.thingsboard.server.common.data.notification.template.NotificationTemplate; +import org.thingsboard.server.common.data.oauth2.OAuth2Client; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.data.security.UserCredentials; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AttributeDeleteMsg; +import org.thingsboard.server.gen.edge.v1.CustomerUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; +import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; +import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; +import org.thingsboard.server.gen.edge.v1.EntityDataProto; +import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; +import org.thingsboard.server.gen.edge.v1.NotificationRuleUpdateMsg; +import org.thingsboard.server.gen.edge.v1.NotificationTargetUpdateMsg; +import org.thingsboard.server.gen.edge.v1.NotificationTemplateUpdateMsg; +import org.thingsboard.server.gen.edge.v1.OAuth2ClientUpdateMsg; +import org.thingsboard.server.gen.edge.v1.OAuth2DomainUpdateMsg; +import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; +import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg; +import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; +import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; +import org.thingsboard.server.gen.edge.v1.RpcRequestMsg; +import org.thingsboard.server.gen.edge.v1.RpcResponseMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; +import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; +import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; +import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; +import org.thingsboard.server.gen.edge.v1.WidgetsBundleUpdateMsg; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.List; +import java.util.UUID; + +@Slf4j +public class EdgeMsgConstructorUtils { + + public static AlarmUpdateMsg constructAlarmUpdatedMsg(UpdateMsgType msgType, Alarm alarm) { + return AlarmUpdateMsg.newBuilder().setMsgType(msgType) + .setEntity(JacksonUtil.toString(alarm)) + .setIdMSB(alarm.getId().getId().getMostSignificantBits()) + .setIdLSB(alarm.getId().getId().getLeastSignificantBits()).build(); + } + + public static AlarmCommentUpdateMsg constructAlarmCommentUpdatedMsg(UpdateMsgType msgType, AlarmComment alarmComment) { + return AlarmCommentUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(alarmComment)).build(); + } + + public static AssetUpdateMsg constructAssetUpdatedMsg(UpdateMsgType msgType, Asset asset) { + return AssetUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(asset)) + .setIdMSB(asset.getUuidId().getMostSignificantBits()) + .setIdLSB(asset.getUuidId().getLeastSignificantBits()).build(); + } + + public static AssetUpdateMsg constructAssetDeleteMsg(AssetId assetId) { + return AssetUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(assetId.getId().getMostSignificantBits()) + .setIdLSB(assetId.getId().getLeastSignificantBits()).build(); + } + + public static AssetProfileUpdateMsg constructAssetProfileUpdatedMsg(UpdateMsgType msgType, AssetProfile assetProfile) { + return AssetProfileUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(assetProfile)) + .setIdMSB(assetProfile.getId().getId().getMostSignificantBits()) + .setIdLSB(assetProfile.getId().getId().getLeastSignificantBits()).build(); + } + + public static AssetProfileUpdateMsg constructAssetProfileDeleteMsg(AssetProfileId assetProfileId) { + return AssetProfileUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(assetProfileId.getId().getMostSignificantBits()) + .setIdLSB(assetProfileId.getId().getLeastSignificantBits()).build(); + } + + public static CustomerUpdateMsg constructCustomerUpdatedMsg(UpdateMsgType msgType, Customer customer) { + return CustomerUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(customer)) + .setIdMSB(customer.getId().getId().getMostSignificantBits()) + .setIdLSB(customer.getId().getId().getLeastSignificantBits()).build(); + } + + public static CustomerUpdateMsg constructCustomerDeleteMsg(CustomerId customerId) { + return CustomerUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(customerId.getId().getMostSignificantBits()) + .setIdLSB(customerId.getId().getLeastSignificantBits()).build(); + } + + public static DashboardUpdateMsg constructDashboardUpdatedMsg(UpdateMsgType msgType, Dashboard dashboard) { + return DashboardUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(dashboard)) + .setIdMSB(dashboard.getId().getId().getMostSignificantBits()) + .setIdLSB(dashboard.getId().getId().getLeastSignificantBits()).build(); + } + + public static DashboardUpdateMsg constructDashboardDeleteMsg(DashboardId dashboardId) { + return DashboardUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(dashboardId.getId().getMostSignificantBits()) + .setIdLSB(dashboardId.getId().getLeastSignificantBits()).build(); + } + + public static DeviceUpdateMsg constructDeviceUpdatedMsg(UpdateMsgType msgType, Device device) { + return DeviceUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(device)) + .setIdMSB(device.getId().getId().getMostSignificantBits()) + .setIdLSB(device.getId().getId().getLeastSignificantBits()).build(); + } + + public static DeviceUpdateMsg constructDeviceDeleteMsg(DeviceId deviceId) { + return DeviceUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(deviceId.getId().getMostSignificantBits()) + .setIdLSB(deviceId.getId().getLeastSignificantBits()).build(); + } + + public static DeviceCredentialsUpdateMsg constructDeviceCredentialsUpdatedMsg(DeviceCredentials deviceCredentials) { + return DeviceCredentialsUpdateMsg.newBuilder().setEntity(JacksonUtil.toString(deviceCredentials)).build(); + } + + public static DeviceProfileUpdateMsg constructDeviceProfileUpdatedMsg(UpdateMsgType msgType, DeviceProfile deviceProfile) { + return DeviceProfileUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(deviceProfile)) + .setIdMSB(deviceProfile.getId().getId().getMostSignificantBits()) + .setIdLSB(deviceProfile.getId().getId().getLeastSignificantBits()).build(); + } + + public static DeviceProfileUpdateMsg constructDeviceProfileDeleteMsg(DeviceProfileId deviceProfileId) { + return DeviceProfileUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(deviceProfileId.getId().getMostSignificantBits()) + .setIdLSB(deviceProfileId.getId().getLeastSignificantBits()).build(); + } + + public static DeviceRpcCallMsg constructDeviceRpcCallMsg(UUID deviceId, JsonNode body) { + DeviceRpcCallMsg.Builder builder = constructDeviceRpcMsg(deviceId, body); + if (body.has("error") || body.has("response")) { + RpcResponseMsg.Builder responseBuilder = RpcResponseMsg.newBuilder(); + if (body.has("error")) { + responseBuilder.setError(body.get("error").asText()); + } else { + responseBuilder.setResponse(body.get("response").asText()); + } + builder.setResponseMsg(responseBuilder.build()); + } else { + RpcRequestMsg.Builder requestBuilder = RpcRequestMsg.newBuilder(); + requestBuilder.setMethod(body.get("method").asText()); + requestBuilder.setParams(body.get("params").asText()); + builder.setRequestMsg(requestBuilder.build()); + } + return builder.build(); + } + + private static DeviceRpcCallMsg.Builder constructDeviceRpcMsg(UUID deviceId, JsonNode body) { + DeviceRpcCallMsg.Builder builder = DeviceRpcCallMsg.newBuilder() + .setDeviceIdMSB(deviceId.getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getLeastSignificantBits()) + .setRequestId(body.get("requestId").asInt()); + if (body.get("oneway") != null) { + builder.setOneway(body.get("oneway").asBoolean()); + } + if (body.get("requestUUID") != null) { + UUID requestUUID = UUID.fromString(body.get("requestUUID").asText()); + builder.setRequestUuidMSB(requestUUID.getMostSignificantBits()) + .setRequestUuidLSB(requestUUID.getLeastSignificantBits()); + } + if (body.get("expirationTime") != null) { + builder.setExpirationTime(body.get("expirationTime").asLong()); + } + if (body.get("persisted") != null) { + builder.setPersisted(body.get("persisted").asBoolean()); + } + if (body.get("retries") != null) { + builder.setRetries(body.get("retries").asInt()); + } + if (body.get("additionalInfo") != null) { + builder.setAdditionalInfo(JacksonUtil.toString(body.get("additionalInfo"))); + } + if (body.get("serviceId") != null) { + builder.setServiceId(body.get("serviceId").asText()); + } + if (body.get("sessionId") != null) { + builder.setSessionId(body.get("sessionId").asText()); + } + return builder; + } + + public static EdgeConfiguration constructEdgeConfiguration(Edge edge) { + EdgeConfiguration.Builder builder = EdgeConfiguration.newBuilder() + .setEdgeIdMSB(edge.getId().getId().getMostSignificantBits()) + .setEdgeIdLSB(edge.getId().getId().getLeastSignificantBits()) + .setTenantIdMSB(edge.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(edge.getTenantId().getId().getLeastSignificantBits()) + .setName(edge.getName()) + .setType(edge.getType()) + .setRoutingKey(edge.getRoutingKey()) + .setSecret(edge.getSecret()) + .setAdditionalInfo(JacksonUtil.toString(edge.getAdditionalInfo())) + .setCloudType("CE"); + if (edge.getCustomerId() != null) { + builder.setCustomerIdMSB(edge.getCustomerId().getId().getMostSignificantBits()) + .setCustomerIdLSB(edge.getCustomerId().getId().getLeastSignificantBits()); + } + return builder.build(); + } + + public static EntityViewUpdateMsg constructEntityViewUpdatedMsg(UpdateMsgType msgType, EntityView entityView) { + return EntityViewUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(entityView)) + .setIdMSB(entityView.getId().getId().getMostSignificantBits()) + .setIdLSB(entityView.getId().getId().getLeastSignificantBits()).build(); + } + + public static EntityViewUpdateMsg constructEntityViewDeleteMsg(EntityViewId entityViewId) { + return EntityViewUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(entityViewId.getId().getMostSignificantBits()) + .setIdLSB(entityViewId.getId().getLeastSignificantBits()).build(); + } + + public static NotificationRuleUpdateMsg constructNotificationRuleUpdateMsg(UpdateMsgType msgType, NotificationRule notificationRule) { + return NotificationRuleUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(notificationRule)).build(); + } + + public static NotificationRuleUpdateMsg constructNotificationRuleDeleteMsg(NotificationRuleId notificationRuleId) { + return NotificationRuleUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(notificationRuleId.getId().getMostSignificantBits()) + .setIdLSB(notificationRuleId.getId().getLeastSignificantBits()).build(); + } + + public static NotificationTargetUpdateMsg constructNotificationTargetUpdateMsg(UpdateMsgType msgType, NotificationTarget notificationTarget) { + return NotificationTargetUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(notificationTarget)).build(); + } + + public static NotificationTargetUpdateMsg constructNotificationTargetDeleteMsg(NotificationTargetId notificationTargetId) { + return NotificationTargetUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(notificationTargetId.getId().getMostSignificantBits()) + .setIdLSB(notificationTargetId.getId().getLeastSignificantBits()).build(); + } + + public static NotificationTemplateUpdateMsg constructNotificationTemplateUpdateMsg(UpdateMsgType msgType, NotificationTemplate notificationTemplate) { + return NotificationTemplateUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(notificationTemplate)).build(); + } + + public static NotificationTemplateUpdateMsg constructNotificationTemplateDeleteMsg(NotificationTemplateId notificationTemplateId) { + return NotificationTemplateUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(notificationTemplateId.getId().getMostSignificantBits()) + .setIdLSB(notificationTemplateId.getId().getLeastSignificantBits()).build(); + } + + public static OAuth2ClientUpdateMsg constructOAuth2ClientUpdateMsg(UpdateMsgType msgType, OAuth2Client oAuth2Client) { + return OAuth2ClientUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(oAuth2Client)) + .setIdMSB(oAuth2Client.getId().getId().getMostSignificantBits()) + .setIdLSB(oAuth2Client.getId().getId().getLeastSignificantBits()).build(); + } + + public static OAuth2ClientUpdateMsg constructOAuth2ClientDeleteMsg(OAuth2ClientId oAuth2ClientId) { + return OAuth2ClientUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(oAuth2ClientId.getId().getMostSignificantBits()) + .setIdLSB(oAuth2ClientId.getId().getLeastSignificantBits()).build(); + } + + public static OAuth2DomainUpdateMsg constructOAuth2DomainUpdateMsg(UpdateMsgType msgType, DomainInfo domainInfo) { + return OAuth2DomainUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(domainInfo)) + .setIdMSB(domainInfo.getId().getId().getMostSignificantBits()) + .setIdLSB(domainInfo.getId().getId().getLeastSignificantBits()).build(); + } + + public static OAuth2DomainUpdateMsg constructOAuth2DomainDeleteMsg(DomainId domainId) { + return OAuth2DomainUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(domainId.getId().getMostSignificantBits()) + .setIdLSB(domainId.getId().getLeastSignificantBits()) + .build(); + } + + public static OtaPackageUpdateMsg constructOtaPackageUpdatedMsg(UpdateMsgType msgType, OtaPackage otaPackage) { + return OtaPackageUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(otaPackage)) + .setIdMSB(otaPackage.getId().getId().getMostSignificantBits()) + .setIdLSB(otaPackage.getId().getId().getLeastSignificantBits()).build(); + } + + public static OtaPackageUpdateMsg constructOtaPackageDeleteMsg(OtaPackageId otaPackageId) { + return OtaPackageUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(otaPackageId.getId().getMostSignificantBits()) + .setIdLSB(otaPackageId.getId().getLeastSignificantBits()).build(); + } + + public static QueueUpdateMsg constructQueueUpdatedMsg(UpdateMsgType msgType, Queue queue) { + return QueueUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(queue)) + .setIdMSB(queue.getId().getId().getMostSignificantBits()) + .setIdLSB(queue.getId().getId().getLeastSignificantBits()).build(); + } + + public static QueueUpdateMsg constructQueueDeleteMsg(QueueId queueId) { + return QueueUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(queueId.getId().getMostSignificantBits()) + .setIdLSB(queueId.getId().getLeastSignificantBits()).build(); + } + + public static RelationUpdateMsg constructRelationUpdatedMsg(UpdateMsgType msgType, EntityRelation entityRelation) { + return RelationUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(entityRelation)).build(); + } + + public static ResourceUpdateMsg constructResourceUpdatedMsg(UpdateMsgType msgType, TbResource tbResource) { + return ResourceUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(tbResource)) + .setIdMSB(tbResource.getId().getId().getMostSignificantBits()) + .setIdLSB(tbResource.getId().getId().getLeastSignificantBits()).build(); + } + + public static ResourceUpdateMsg constructResourceDeleteMsg(TbResourceId tbResourceId) { + return ResourceUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(tbResourceId.getId().getMostSignificantBits()) + .setIdLSB(tbResourceId.getId().getLeastSignificantBits()).build(); + } + + public static RuleChainUpdateMsg constructRuleChainUpdatedMsg(UpdateMsgType msgType, RuleChain ruleChain, boolean isRoot) { + boolean isTemplateRoot = ruleChain.isRoot(); + ruleChain.setRoot(isRoot); + RuleChainUpdateMsg result = RuleChainUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(ruleChain)) + .setIdMSB(ruleChain.getId().getId().getMostSignificantBits()) + .setIdLSB(ruleChain.getId().getId().getLeastSignificantBits()).build(); + ruleChain.setRoot(isTemplateRoot); + return result; + } + + public static RuleChainUpdateMsg constructRuleChainDeleteMsg(RuleChainId ruleChainId) { + return RuleChainUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(ruleChainId.getId().getMostSignificantBits()) + .setIdLSB(ruleChainId.getId().getLeastSignificantBits()).build(); + } + + public static RuleChainMetadataUpdateMsg constructRuleChainMetadataUpdatedMsg(UpdateMsgType msgType, RuleChainMetaData ruleChainMetaData) { + return RuleChainMetadataUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(ruleChainMetaData)).build(); + } + + public static EntityDataProto constructEntityDataMsg(TenantId tenantId, EntityId entityId, EdgeEventActionType actionType, JsonElement entityData) { + EntityDataProto.Builder builder = EntityDataProto.newBuilder() + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .setEntityType(entityId.getEntityType().name()); + switch (actionType) { + case TIMESERIES_UPDATED: + try { + JsonObject data = entityData.getAsJsonObject(); + long ts; + if (data.get("ts") != null && !data.get("ts").isJsonNull()) { + ts = data.getAsJsonPrimitive("ts").getAsLong(); + } else { + ts = System.currentTimeMillis(); + } + builder.setPostTelemetryMsg(JsonConverter.convertToTelemetryProto(data.getAsJsonObject("data"), ts)); + } catch (Exception e) { + log.warn("[{}][{}] Can't convert to telemetry proto, entityData [{}]", tenantId, entityId, entityData, e); + } + break; + case ATTRIBUTES_UPDATED: + try { + JsonObject data = entityData.getAsJsonObject(); + TransportProtos.PostAttributeMsg attributesUpdatedMsg = JsonConverter.convertToAttributesProto(data.getAsJsonObject("kv")); + builder.setAttributesUpdatedMsg(attributesUpdatedMsg); + builder.setPostAttributeScope(getScopeOfDefault(data)); + } catch (Exception e) { + log.warn("[{}][{}] Can't convert to AttributesUpdatedMsg proto, entityData [{}]", tenantId, entityId, entityData, e); + } + break; + case POST_ATTRIBUTES: + try { + JsonObject data = entityData.getAsJsonObject(); + TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(data.getAsJsonObject("kv")); + builder.setPostAttributesMsg(postAttributesMsg); + builder.setPostAttributeScope(getScopeOfDefault(data)); + } catch (Exception e) { + log.warn("[{}][{}] Can't convert to PostAttributesMsg, entityData [{}]", tenantId, entityId, entityData, e); + } + break; + case ATTRIBUTES_DELETED: + try { + AttributeDeleteMsg.Builder attributeDeleteMsg = AttributeDeleteMsg.newBuilder(); + attributeDeleteMsg.setScope(entityData.getAsJsonObject().getAsJsonPrimitive("scope").getAsString()); + JsonArray jsonArray = entityData.getAsJsonObject().getAsJsonArray("keys"); + List keys = new Gson().fromJson(jsonArray.toString(), new TypeToken<>(){}.getType()); + attributeDeleteMsg.addAllAttributeNames(keys); + attributeDeleteMsg.build(); + builder.setAttributeDeleteMsg(attributeDeleteMsg); + } catch (Exception e) { + log.warn("[{}][{}] Can't convert to AttributeDeleteMsg proto, entityData [{}]", tenantId, entityId, entityData, e); + } + break; + } + return builder.build(); + } + + private static String getScopeOfDefault(JsonObject data) { + JsonPrimitive scope = data.getAsJsonPrimitive("scope"); + String result = DataConstants.SERVER_SCOPE; + if (scope != null && StringUtils.isNotBlank(scope.getAsString())) { + result = scope.getAsString(); + } + return result; + } + + public static TenantUpdateMsg constructTenantUpdateMsg(UpdateMsgType msgType, Tenant tenant) { + return TenantUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(tenant)).build(); + } + + public static TenantProfileUpdateMsg constructTenantProfileUpdateMsg(UpdateMsgType msgType, TenantProfile tenantProfile) { + return TenantProfileUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(tenantProfile)).build(); + } + + public static UserUpdateMsg constructUserUpdatedMsg(UpdateMsgType msgType, User user) { + return UserUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(user)) + .setIdMSB(user.getId().getId().getMostSignificantBits()) + .setIdLSB(user.getId().getId().getLeastSignificantBits()).build(); + } + + public static UserUpdateMsg constructUserDeleteMsg(UserId userId) { + return UserUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(userId.getId().getMostSignificantBits()) + .setIdLSB(userId.getId().getLeastSignificantBits()).build(); + } + + public static UserCredentialsUpdateMsg constructUserCredentialsUpdatedMsg(UserCredentials userCredentials) { + return UserCredentialsUpdateMsg.newBuilder().setEntity(JacksonUtil.toString(userCredentials)).build(); + } + + public static WidgetsBundleUpdateMsg constructWidgetsBundleUpdateMsg(UpdateMsgType msgType, WidgetsBundle widgetsBundle, List widgets) { + return WidgetsBundleUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(widgetsBundle)) + .setWidgets(JacksonUtil.toString(widgets)) + .setIdMSB(widgetsBundle.getId().getId().getMostSignificantBits()) + .setIdLSB(widgetsBundle.getId().getId().getLeastSignificantBits()).build(); + } + + public static WidgetsBundleUpdateMsg constructWidgetsBundleDeleteMsg(WidgetsBundleId widgetsBundleId) { + return WidgetsBundleUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(widgetsBundleId.getId().getMostSignificantBits()) + .setIdLSB(widgetsBundleId.getId().getLeastSignificantBits()) + .build(); + } + + public static WidgetTypeUpdateMsg constructWidgetTypeUpdateMsg(UpdateMsgType msgType, WidgetTypeDetails widgetTypeDetails) { + return WidgetTypeUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(widgetTypeDetails)) + .setIdMSB(widgetTypeDetails.getId().getId().getMostSignificantBits()) + .setIdLSB(widgetTypeDetails.getId().getId().getLeastSignificantBits()).build(); + } + + public static WidgetTypeUpdateMsg constructWidgetTypeDeleteMsg(WidgetTypeId widgetTypeId) { + return WidgetTypeUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(widgetTypeId.getId().getMostSignificantBits()) + .setIdLSB(widgetTypeId.getId().getLeastSignificantBits()) + .build(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 2e9465fc9d..15a4bcaa0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -564,7 +564,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i edgeState.put(ACTIVITY_STATE, false); edgeState.put(LAST_DISCONNECT_TIME, ts); } - ctx.getNotificationRuleProcessor().process(EdgeConnectionTrigger.builder() + ctx.getRuleProcessor().process(EdgeConnectionTrigger.builder() .tenantId(tenantId) .customerId(edge.getCustomerId()) .edgeId(edgeId) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 42a6a60c2d..96ecd78bec 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -78,17 +79,10 @@ import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg; import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg; import org.thingsboard.server.service.edge.EdgeContextComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.fetch.EdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.GeneralEdgeEventFetcher; -import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmProcessor; -import org.thingsboard.server.service.edge.rpc.processor.asset.AssetProcessor; -import org.thingsboard.server.service.edge.rpc.processor.asset.profile.AssetProfileProcessor; -import org.thingsboard.server.service.edge.rpc.processor.dashboard.DashboardProcessor; -import org.thingsboard.server.service.edge.rpc.processor.device.DeviceProcessor; -import org.thingsboard.server.service.edge.rpc.processor.device.profile.DeviceProfileProcessor; -import org.thingsboard.server.service.edge.rpc.processor.entityview.EntityViewProcessor; -import org.thingsboard.server.service.edge.rpc.processor.relation.RelationProcessor; -import org.thingsboard.server.service.edge.rpc.processor.resource.ResourceProcessor; +import org.thingsboard.server.service.edge.rpc.utils.EdgeVersionUtils; import java.io.Closeable; import java.util.ArrayList; @@ -238,7 +232,7 @@ public abstract class EdgeGrpcSession implements Closeable { this.tenantId = edge.getTenantId(); this.edge = edge; EdgeUpdateMsg edgeConfig = EdgeUpdateMsg.newBuilder() - .setConfiguration(ctx.getEdgeMsgConstructor().constructEdgeConfiguration(edge)).build(); + .setConfiguration(EdgeMsgConstructorUtils.constructEdgeConfiguration(edge)).build(); ResponseMsg edgeConfigMsg = ResponseMsg.newBuilder() .setEdgeUpdateMsg(edgeConfig) .build(); @@ -352,13 +346,13 @@ public abstract class EdgeGrpcSession implements Closeable { return ConnectResponseMsg.newBuilder() .setResponseCode(ConnectResponseCode.ACCEPTED) .setErrorMsg("") - .setConfiguration(ctx.getEdgeMsgConstructor().constructEdgeConfiguration(edge)) + .setConfiguration(EdgeMsgConstructorUtils.constructEdgeConfiguration(edge)) .setMaxInboundMessageSize(maxInboundMessageSize) .build(); } String error = "Failed to validate the edge!"; String failureMsg = String.format("%s Provided request secret: %s", error, request.getEdgeSecret()); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) + ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); return ConnectResponseMsg.newBuilder() .setResponseCode(ConnectResponseCode.BAD_CREDENTIALS) @@ -366,7 +360,7 @@ public abstract class EdgeGrpcSession implements Closeable { .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); } catch (Exception e) { String failureMsg = "Failed to process edge connection!"; - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) + ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); log.error(failureMsg, e); return ConnectResponseMsg.newBuilder() @@ -451,7 +445,7 @@ public abstract class EdgeGrpcSession implements Closeable { if (attempt == 2) { // Send a failure notification only on the second attempt. // This ensures that failure alerts are sent just once to avoid redundant notifications. - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) + ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); } log.warn("[{}][{}] {}, attempt: {}", tenantId, sessionId, failureMsg, attempt); @@ -464,7 +458,7 @@ public abstract class EdgeGrpcSession implements Closeable { String message = String.format("Downlink msg size %s exceeds client max inbound message size %s. " + "Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE env variable on the edge and restart it.", downlinkMsg.getSerializedSize(), clientMaxInboundMessageSize); log.error("[{}][{}][{}] {} Message {}", tenantId, edge.getId(), sessionId, message, downlinkMsg); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) + ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(message).error(error).build()); sessionState.getPendingMsgsMap().remove(downlinkMsg.getDownlinkMsgId()); } else { @@ -479,7 +473,7 @@ public abstract class EdgeGrpcSession implements Closeable { String failureMsg = String.format("Failed to deliver messages: %s", copy); log.warn("[{}][{}] Failed to deliver the batch after {} attempts. Next messages are going to be discarded {}", tenantId, sessionId, MAX_DOWNLINK_ATTEMPTS, copy); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) + ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg) .error("Failed to deliver messages after " + MAX_DOWNLINK_ATTEMPTS + " attempts").build()); stopCurrentSendDownlinkMsgsTask(false); @@ -752,39 +746,12 @@ public abstract class EdgeGrpcSession implements Closeable { protected DownlinkMsg convertEntityEventToDownlink(EdgeEvent edgeEvent) { log.trace("[{}] Executing convertEntityEventToDownlink, edgeEvent [{}], action [{}]", edgeEvent.getTenantId(), edgeEvent, edgeEvent.getAction()); - return switch (edgeEvent.getType()) { - case EDGE -> ctx.getEdgeProcessor().convertEdgeEventToDownlink(edgeEvent); - case DEVICE -> ctx.getDeviceProcessor().convertDeviceEventToDownlink(edgeEvent, edgeVersion); - case DEVICE_PROFILE -> ctx.getDeviceProfileProcessor().convertDeviceProfileEventToDownlink(edgeEvent, edgeVersion); - case ASSET_PROFILE -> ctx.getAssetProfileProcessor().convertAssetProfileEventToDownlink(edgeEvent, edgeVersion); - case ASSET -> ctx.getAssetProcessor().convertAssetEventToDownlink(edgeEvent, edgeVersion); - case ENTITY_VIEW -> ctx.getEntityViewProcessor().convertEntityViewEventToDownlink(edgeEvent, edgeVersion); - case DASHBOARD -> ctx.getDashboardProcessor().convertDashboardEventToDownlink(edgeEvent, edgeVersion); - case CUSTOMER -> ctx.getCustomerProcessor().convertCustomerEventToDownlink(edgeEvent, edgeVersion); - case RULE_CHAIN -> ctx.getRuleChainProcessor().convertRuleChainEventToDownlink(edgeEvent, edgeVersion); - case RULE_CHAIN_METADATA -> ctx.getRuleChainProcessor().convertRuleChainMetadataEventToDownlink(edgeEvent, edgeVersion); - case ALARM -> ctx.getAlarmProcessor().convertAlarmEventToDownlink(edgeEvent, edgeVersion); - case ALARM_COMMENT -> ctx.getAlarmProcessor().convertAlarmCommentEventToDownlink(edgeEvent, edgeVersion); - case USER -> ctx.getUserProcessor().convertUserEventToDownlink(edgeEvent, edgeVersion); - case RELATION -> ctx.getRelationProcessor().convertRelationEventToDownlink(edgeEvent, edgeVersion); - case WIDGETS_BUNDLE -> ctx.getWidgetBundleProcessor().convertWidgetsBundleEventToDownlink(edgeEvent, edgeVersion); - case WIDGET_TYPE -> ctx.getWidgetTypeProcessor().convertWidgetTypeEventToDownlink(edgeEvent, edgeVersion); - case ADMIN_SETTINGS -> ctx.getAdminSettingsProcessor().convertAdminSettingsEventToDownlink(edgeEvent, edgeVersion); - case OTA_PACKAGE -> ctx.getOtaPackageProcessor().convertOtaPackageEventToDownlink(edgeEvent, edgeVersion); - case TB_RESOURCE -> ctx.getResourceProcessor().convertResourceEventToDownlink(edgeEvent, edgeVersion); - case QUEUE -> ctx.getQueueProcessor().convertQueueEventToDownlink(edgeEvent, edgeVersion); - case TENANT -> ctx.getTenantProcessor().convertTenantEventToDownlink(edgeEvent, edgeVersion); - case TENANT_PROFILE -> ctx.getTenantProfileProcessor().convertTenantProfileEventToDownlink(edgeEvent, edgeVersion); - case NOTIFICATION_RULE -> ctx.getNotificationEdgeProcessor().convertNotificationRuleToDownlink(edgeEvent); - case NOTIFICATION_TARGET -> ctx.getNotificationEdgeProcessor().convertNotificationTargetToDownlink(edgeEvent); - case NOTIFICATION_TEMPLATE -> ctx.getNotificationEdgeProcessor().convertNotificationTemplateToDownlink(edgeEvent); - case OAUTH2_CLIENT -> ctx.getOAuth2EdgeProcessor().convertOAuth2ClientEventToDownlink(edgeEvent, edgeVersion); - case DOMAIN -> ctx.getOAuth2EdgeProcessor().convertOAuth2DomainEventToDownlink(edgeEvent, edgeVersion); - default -> { - log.warn("[{}] Unsupported edge event type [{}]", edgeEvent.getTenantId(), edgeEvent); - yield null; - } - }; + if ((EdgeEventType.OAUTH2_CLIENT.equals(edgeEvent.getType()) || EdgeEventType.DOMAIN.equals(edgeEvent.getType())) && + (EdgeVersionUtils.isEdgeVersionOlderThan(edgeVersion, EdgeVersion.V_3_8_0))) { + return null; + } + + return ctx.getProcessor(edgeEvent.getType()).convertEdgeEventToDownlink(edgeEvent); } public void addEventToHighPriorityQueue(EdgeEvent edgeEvent) { @@ -803,38 +770,32 @@ public abstract class EdgeGrpcSession implements Closeable { try { if (uplinkMsg.getDeviceProfileUpdateMsgCount() > 0) { for (DeviceProfileUpdateMsg deviceProfileUpdateMsg : uplinkMsg.getDeviceProfileUpdateMsgList()) { - result.add(((DeviceProfileProcessor) ctx.getDeviceProfileEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processDeviceProfileMsgFromEdge(edge.getTenantId(), edge, deviceProfileUpdateMsg)); + result.add(ctx.getDeviceProfileProcessor().processDeviceProfileMsgFromEdge(edge.getTenantId(), edge, deviceProfileUpdateMsg)); } } if (uplinkMsg.getDeviceUpdateMsgCount() > 0) { for (DeviceUpdateMsg deviceUpdateMsg : uplinkMsg.getDeviceUpdateMsgList()) { - result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processDeviceMsgFromEdge(edge.getTenantId(), edge, deviceUpdateMsg)); + result.add(ctx.getDeviceProcessor().processDeviceMsgFromEdge(edge.getTenantId(), edge, deviceUpdateMsg)); } } if (uplinkMsg.getDeviceCredentialsUpdateMsgCount() > 0) { for (DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg : uplinkMsg.getDeviceCredentialsUpdateMsgList()) { - result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processDeviceCredentialsMsgFromEdge(edge.getTenantId(), edge.getId(), deviceCredentialsUpdateMsg)); + result.add(ctx.getDeviceProcessor().processDeviceCredentialsMsgFromEdge(edge.getTenantId(), edge.getId(), deviceCredentialsUpdateMsg)); } } if (uplinkMsg.getAssetProfileUpdateMsgCount() > 0) { for (AssetProfileUpdateMsg assetProfileUpdateMsg : uplinkMsg.getAssetProfileUpdateMsgList()) { - result.add(((AssetProfileProcessor) ctx.getAssetProfileEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processAssetProfileMsgFromEdge(edge.getTenantId(), edge, assetProfileUpdateMsg)); + result.add(ctx.getAssetProfileProcessor().processAssetProfileMsgFromEdge(edge.getTenantId(), edge, assetProfileUpdateMsg)); } } if (uplinkMsg.getAssetUpdateMsgCount() > 0) { for (AssetUpdateMsg assetUpdateMsg : uplinkMsg.getAssetUpdateMsgList()) { - result.add(((AssetProcessor) ctx.getAssetEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg)); + result.add(ctx.getAssetProcessor().processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg)); } } if (uplinkMsg.getEntityViewUpdateMsgCount() > 0) { for (EntityViewUpdateMsg entityViewUpdateMsg : uplinkMsg.getEntityViewUpdateMsgList()) { - result.add(((EntityViewProcessor) ctx.getEntityViewProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg)); + result.add(ctx.getEntityViewProcessor().processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg)); } } if (uplinkMsg.getEntityDataCount() > 0) { @@ -844,32 +805,27 @@ public abstract class EdgeGrpcSession implements Closeable { } if (uplinkMsg.getAlarmUpdateMsgCount() > 0) { for (AlarmUpdateMsg alarmUpdateMsg : uplinkMsg.getAlarmUpdateMsgList()) { - result.add(((AlarmProcessor) ctx.getAlarmEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processAlarmMsgFromEdge(edge.getTenantId(), edge.getId(), alarmUpdateMsg)); + result.add(ctx.getAlarmProcessor().processAlarmMsgFromEdge(edge.getTenantId(), edge.getId(), alarmUpdateMsg)); } } if (uplinkMsg.getAlarmCommentUpdateMsgCount() > 0) { for (AlarmCommentUpdateMsg alarmCommentUpdateMsg : uplinkMsg.getAlarmCommentUpdateMsgList()) { - result.add(((AlarmProcessor) ctx.getAlarmEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processAlarmCommentMsgFromEdge(edge.getTenantId(), edge.getId(), alarmCommentUpdateMsg)); + result.add(ctx.getAlarmCommentProcessor().processAlarmCommentMsgFromEdge(edge.getTenantId(), edge.getId(), alarmCommentUpdateMsg)); } } if (uplinkMsg.getRelationUpdateMsgCount() > 0) { for (RelationUpdateMsg relationUpdateMsg : uplinkMsg.getRelationUpdateMsgList()) { - result.add(((RelationProcessor) ctx.getRelationEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processRelationMsgFromEdge(edge.getTenantId(), edge, relationUpdateMsg)); + result.add(ctx.getRelationProcessor().processRelationMsgFromEdge(edge.getTenantId(), edge, relationUpdateMsg)); } } if (uplinkMsg.getDashboardUpdateMsgCount() > 0) { for (DashboardUpdateMsg dashboardUpdateMsg : uplinkMsg.getDashboardUpdateMsgList()) { - result.add(((DashboardProcessor) ctx.getDashboardEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processDashboardMsgFromEdge(edge.getTenantId(), edge, dashboardUpdateMsg)); + result.add(ctx.getDashboardProcessor().processDashboardMsgFromEdge(edge.getTenantId(), edge, dashboardUpdateMsg)); } } if (uplinkMsg.getResourceUpdateMsgCount() > 0) { for (ResourceUpdateMsg resourceUpdateMsg : uplinkMsg.getResourceUpdateMsgList()) { - result.add(((ResourceProcessor) ctx.getResourceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processResourceMsgFromEdge(edge.getTenantId(), edge, resourceUpdateMsg)); + result.add(ctx.getResourceProcessor().processResourceMsgFromEdge(edge.getTenantId(), edge, resourceUpdateMsg)); } } if (uplinkMsg.getRuleChainMetadataRequestMsgCount() > 0) { @@ -899,8 +855,7 @@ public abstract class EdgeGrpcSession implements Closeable { } if (uplinkMsg.getDeviceRpcCallMsgCount() > 0) { for (DeviceRpcCallMsg deviceRpcCallMsg : uplinkMsg.getDeviceRpcCallMsgList()) { - result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processDeviceRpcCallFromEdge(edge.getTenantId(), edge, deviceRpcCallMsg)); + result.add(ctx.getDeviceProcessor().processDeviceRpcCallFromEdge(edge.getTenantId(), edge, deviceRpcCallMsg)); } } if (uplinkMsg.getWidgetBundleTypesRequestMsgCount() > 0) { @@ -916,7 +871,7 @@ public abstract class EdgeGrpcSession implements Closeable { } catch (Exception e) { String failureMsg = String.format("Can't process uplink msg [%s] from edge", uplinkMsg); log.error("[{}][{}] Can't process uplink msg [{}]", edge.getTenantId(), sessionId, uplinkMsg, e); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(edge.getTenantId()).edgeId(edge.getId()) + ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(edge.getTenantId()).edgeId(edge.getId()) .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); return Futures.immediateFailedFuture(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/BaseMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/BaseMsgConstructorFactory.java deleted file mode 100644 index dd76f97d8f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/BaseMsgConstructorFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public abstract class BaseMsgConstructorFactory { - - @Autowired - protected T v1Constructor; - - @Autowired - protected U v2Constructor; - - public MsgConstructor getMsgConstructorByEdgeVersion(EdgeVersion edgeVersion) { - return switch (edgeVersion) { - case V_3_3_0, V_3_3_3, V_3_4_0, V_3_6_0, V_3_6_1 -> v1Constructor; - default -> v2Constructor; - }; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/MsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/MsgConstructor.java deleted file mode 100644 index c751c6259c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/MsgConstructor.java +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor; - -public interface MsgConstructor { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructor.java deleted file mode 100644 index b5a0f20d6b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructor.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.alarm; - -import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmComment; -import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface AlarmMsgConstructor extends MsgConstructor { - - AlarmUpdateMsg constructAlarmUpdatedMsg(UpdateMsgType msgType, Alarm alarm, String entityName); - - AlarmCommentUpdateMsg constructAlarmCommentUpdatedMsg(UpdateMsgType msgType, AlarmComment alarmComment); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorFactory.java deleted file mode 100644 index ec2e36d431..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.alarm; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class AlarmMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorV1.java deleted file mode 100644 index 81a489671f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorV1.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.alarm; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class AlarmMsgConstructorV1 extends BaseAlarmMsgConstructor { - - @Override - public AlarmUpdateMsg constructAlarmUpdatedMsg(UpdateMsgType msgType, Alarm alarm, String entityName) { - return AlarmUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(alarm.getId().getId().getMostSignificantBits()) - .setIdLSB(alarm.getId().getId().getLeastSignificantBits()) - .setName(alarm.getName()) - .setType(alarm.getType()) - .setOriginatorName(entityName) - .setOriginatorType(alarm.getOriginator().getEntityType().name()) - .setSeverity(alarm.getSeverity().name()) - .setStatus(alarm.getStatus().name()) - .setStartTs(alarm.getStartTs()) - .setEndTs(alarm.getEndTs()) - .setAckTs(alarm.getAckTs()) - .setClearTs(alarm.getClearTs()) - .setDetails(JacksonUtil.toString(alarm.getDetails())) - .setPropagate(alarm.isPropagate()) - .setPropagateToOwner(alarm.isPropagateToOwner()) - .setPropagateToTenant(alarm.isPropagateToTenant()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorV2.java deleted file mode 100644 index 143a41b73c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/AlarmMsgConstructorV2.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.alarm; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class AlarmMsgConstructorV2 extends BaseAlarmMsgConstructor { - - @Override - public AlarmUpdateMsg constructAlarmUpdatedMsg(UpdateMsgType msgType, Alarm alarm, String entityName) { - return AlarmUpdateMsg.newBuilder().setMsgType(msgType) - .setEntity(JacksonUtil.toString(alarm)) - .setIdMSB(alarm.getId().getId().getMostSignificantBits()) - .setIdLSB(alarm.getId().getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructor.java deleted file mode 100644 index ba7b2a81ef..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructor.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.asset; - -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface AssetMsgConstructor extends MsgConstructor { - - AssetUpdateMsg constructAssetUpdatedMsg(UpdateMsgType msgType, Asset asset); - - AssetUpdateMsg constructAssetDeleteMsg(AssetId assetId); - - AssetProfileUpdateMsg constructAssetProfileUpdatedMsg(UpdateMsgType msgType, AssetProfile assetProfile); - - AssetProfileUpdateMsg constructAssetProfileDeleteMsg(AssetProfileId assetProfileId); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorFactory.java deleted file mode 100644 index 1d236fce00..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.asset; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class AssetMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV1.java deleted file mode 100644 index 524edbb5a5..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV1.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.asset; - -import com.google.protobuf.ByteString; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.dao.resource.ImageService; -import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.nio.charset.StandardCharsets; - -@Component -@TbCoreComponent -public class AssetMsgConstructorV1 extends BaseAssetMsgConstructor { - - @Autowired - private ImageService imageService; - - @Override - public AssetUpdateMsg constructAssetUpdatedMsg(UpdateMsgType msgType, Asset asset) { - AssetUpdateMsg.Builder builder = AssetUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(asset.getUuidId().getMostSignificantBits()) - .setIdLSB(asset.getUuidId().getLeastSignificantBits()) - .setName(asset.getName()) - .setType(asset.getType()); - if (asset.getLabel() != null) { - builder.setLabel(asset.getLabel()); - } - if (asset.getCustomerId() != null) { - builder.setCustomerIdMSB(asset.getCustomerId().getId().getMostSignificantBits()); - builder.setCustomerIdLSB(asset.getCustomerId().getId().getLeastSignificantBits()); - } - if (asset.getAssetProfileId() != null) { - builder.setAssetProfileIdMSB(asset.getAssetProfileId().getId().getMostSignificantBits()); - builder.setAssetProfileIdLSB(asset.getAssetProfileId().getId().getLeastSignificantBits()); - } - if (asset.getAdditionalInfo() != null) { - builder.setAdditionalInfo(JacksonUtil.toString(asset.getAdditionalInfo())); - } - return builder.build(); - } - - @Override - public AssetProfileUpdateMsg constructAssetProfileUpdatedMsg(UpdateMsgType msgType, AssetProfile assetProfile) { - assetProfile = JacksonUtil.clone(assetProfile); - imageService.inlineImageForEdge(assetProfile); - AssetProfileUpdateMsg.Builder builder = AssetProfileUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(assetProfile.getId().getId().getMostSignificantBits()) - .setIdLSB(assetProfile.getId().getId().getLeastSignificantBits()) - .setName(assetProfile.getName()) - .setDefault(assetProfile.isDefault()); - if (assetProfile.getDefaultDashboardId() != null) { - builder.setDefaultDashboardIdMSB(assetProfile.getDefaultDashboardId().getId().getMostSignificantBits()) - .setDefaultDashboardIdLSB(assetProfile.getDefaultDashboardId().getId().getLeastSignificantBits()); - } - if (assetProfile.getDefaultQueueName() != null) { - builder.setDefaultQueueName(assetProfile.getDefaultQueueName()); - } - if (assetProfile.getDescription() != null) { - builder.setDescription(assetProfile.getDescription()); - } - if (assetProfile.getImage() != null) { - builder.setImage(ByteString.copyFrom(assetProfile.getImage().getBytes(StandardCharsets.UTF_8))); - } - if (assetProfile.getDefaultEdgeRuleChainId() != null) { - builder.setDefaultRuleChainIdMSB(assetProfile.getDefaultEdgeRuleChainId().getId().getMostSignificantBits()) - .setDefaultRuleChainIdLSB(assetProfile.getDefaultEdgeRuleChainId().getId().getLeastSignificantBits()); - } - return builder.build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV2.java deleted file mode 100644 index b48cacb3ee..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV2.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.asset; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class AssetMsgConstructorV2 extends BaseAssetMsgConstructor { - - @Override - public AssetUpdateMsg constructAssetUpdatedMsg(UpdateMsgType msgType, Asset asset) { - return AssetUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(asset)) - .setIdMSB(asset.getUuidId().getMostSignificantBits()) - .setIdLSB(asset.getUuidId().getLeastSignificantBits()).build(); - } - - @Override - public AssetProfileUpdateMsg constructAssetProfileUpdatedMsg(UpdateMsgType msgType, AssetProfile assetProfile) { - return AssetProfileUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(assetProfile)) - .setIdMSB(assetProfile.getId().getId().getMostSignificantBits()) - .setIdLSB(assetProfile.getId().getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/BaseAssetMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/BaseAssetMsgConstructor.java deleted file mode 100644 index a067d59ab4..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/BaseAssetMsgConstructor.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.asset; - -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -public abstract class BaseAssetMsgConstructor implements AssetMsgConstructor { - - @Override - public AssetUpdateMsg constructAssetDeleteMsg(AssetId assetId) { - return AssetUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(assetId.getId().getMostSignificantBits()) - .setIdLSB(assetId.getId().getLeastSignificantBits()).build(); - } - - @Override - public AssetProfileUpdateMsg constructAssetProfileDeleteMsg(AssetProfileId assetProfileId) { - return AssetProfileUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(assetProfileId.getId().getMostSignificantBits()) - .setIdLSB(assetProfileId.getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/BaseCustomerMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/BaseCustomerMsgConstructor.java deleted file mode 100644 index 5e02200a16..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/BaseCustomerMsgConstructor.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.customer; - -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.gen.edge.v1.CustomerUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -public abstract class BaseCustomerMsgConstructor implements CustomerMsgConstructor { - - @Override - public CustomerUpdateMsg constructCustomerDeleteMsg(CustomerId customerId) { - return CustomerUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(customerId.getId().getMostSignificantBits()) - .setIdLSB(customerId.getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructor.java deleted file mode 100644 index 289056fadf..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructor.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.customer; - -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.gen.edge.v1.CustomerUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface CustomerMsgConstructor extends MsgConstructor { - - CustomerUpdateMsg constructCustomerUpdatedMsg(UpdateMsgType msgType, Customer customer); - - CustomerUpdateMsg constructCustomerDeleteMsg(CustomerId customerId); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorFactory.java deleted file mode 100644 index 01df167527..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.customer; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class CustomerMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorV1.java deleted file mode 100644 index 1018997f47..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorV1.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.customer; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.gen.edge.v1.CustomerUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class CustomerMsgConstructorV1 extends BaseCustomerMsgConstructor { - - @Override - public CustomerUpdateMsg constructCustomerUpdatedMsg(UpdateMsgType msgType, Customer customer) { - CustomerUpdateMsg.Builder builder = CustomerUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(customer.getId().getId().getMostSignificantBits()) - .setIdLSB(customer.getId().getId().getLeastSignificantBits()) - .setTitle(customer.getTitle()); - if (customer.getCountry() != null) { - builder.setCountry(customer.getCountry()); - } - if (customer.getState() != null) { - builder.setState(customer.getState()); - } - if (customer.getCity() != null) { - builder.setCity(customer.getCity()); - } - if (customer.getAddress() != null) { - builder.setAddress(customer.getAddress()); - } - if (customer.getAddress2() != null) { - builder.setAddress2(customer.getAddress2()); - } - if (customer.getZip() != null) { - builder.setZip(customer.getZip()); - } - if (customer.getPhone() != null) { - builder.setPhone(customer.getPhone()); - } - if (customer.getEmail() != null) { - builder.setEmail(customer.getEmail()); - } - if (customer.getAdditionalInfo() != null) { - builder.setAdditionalInfo(JacksonUtil.toString(customer.getAdditionalInfo())); - } - return builder.build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorV2.java deleted file mode 100644 index 8f5a99f2ca..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/customer/CustomerMsgConstructorV2.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.customer; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.gen.edge.v1.CustomerUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class CustomerMsgConstructorV2 extends BaseCustomerMsgConstructor { - - @Override - public CustomerUpdateMsg constructCustomerUpdatedMsg(UpdateMsgType msgType, Customer customer) { - return CustomerUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(customer)) - .setIdMSB(customer.getId().getId().getMostSignificantBits()) - .setIdLSB(customer.getId().getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/BaseDashboardMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/BaseDashboardMsgConstructor.java deleted file mode 100644 index f1314d9587..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/BaseDashboardMsgConstructor.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.dashboard; - -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -public abstract class BaseDashboardMsgConstructor implements DashboardMsgConstructor { - - @Override - public DashboardUpdateMsg constructDashboardDeleteMsg(DashboardId dashboardId) { - return DashboardUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(dashboardId.getId().getMostSignificantBits()) - .setIdLSB(dashboardId.getId().getLeastSignificantBits()).build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructor.java deleted file mode 100644 index bdf4968692..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructor.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.dashboard; - -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface DashboardMsgConstructor extends MsgConstructor { - - DashboardUpdateMsg constructDashboardUpdatedMsg(UpdateMsgType msgType, Dashboard dashboard); - - DashboardUpdateMsg constructDashboardDeleteMsg(DashboardId dashboardId); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorFactory.java deleted file mode 100644 index b2d1e789d9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.dashboard; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class DashboardMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV1.java deleted file mode 100644 index ec2d216a03..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV1.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.dashboard; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.dao.resource.ImageService; -import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class DashboardMsgConstructorV1 extends BaseDashboardMsgConstructor { - - @Autowired - private ImageService imageService; - - @Override - public DashboardUpdateMsg constructDashboardUpdatedMsg(UpdateMsgType msgType, Dashboard dashboard) { - dashboard = JacksonUtil.clone(dashboard); - imageService.inlineImagesForEdge(dashboard); - DashboardUpdateMsg.Builder builder = DashboardUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(dashboard.getId().getId().getMostSignificantBits()) - .setIdLSB(dashboard.getId().getId().getLeastSignificantBits()) - .setTitle(dashboard.getTitle()) - .setConfiguration(JacksonUtil.toString(dashboard.getConfiguration())) - .setMobileHide(dashboard.isMobileHide()); - if (dashboard.getAssignedCustomers() != null) { - builder.setAssignedCustomers(JacksonUtil.toString(dashboard.getAssignedCustomers())); - } - if (dashboard.getImage() != null) { - builder.setImage(dashboard.getImage()); - } - if (dashboard.getMobileOrder() != null) { - builder.setMobileOrder(dashboard.getMobileOrder()); - } - return builder.build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV2.java deleted file mode 100644 index d1fb5ae703..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV2.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.dashboard; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class DashboardMsgConstructorV2 extends BaseDashboardMsgConstructor { - - @Override - public DashboardUpdateMsg constructDashboardUpdatedMsg(UpdateMsgType msgType, Dashboard dashboard) { - return DashboardUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(dashboard)) - .setIdMSB(dashboard.getId().getId().getMostSignificantBits()) - .setIdLSB(dashboard.getId().getId().getLeastSignificantBits()).build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/BaseDeviceMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/BaseDeviceMsgConstructor.java deleted file mode 100644 index b875e1c947..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/BaseDeviceMsgConstructor.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.device; - -import com.fasterxml.jackson.databind.JsonNode; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; -import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.RpcRequestMsg; -import org.thingsboard.server.gen.edge.v1.RpcResponseMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -import java.util.UUID; - -public abstract class BaseDeviceMsgConstructor implements DeviceMsgConstructor { - - @Override - public DeviceUpdateMsg constructDeviceDeleteMsg(DeviceId deviceId) { - return DeviceUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(deviceId.getId().getMostSignificantBits()) - .setIdLSB(deviceId.getId().getLeastSignificantBits()).build(); - } - - @Override - public DeviceProfileUpdateMsg constructDeviceProfileDeleteMsg(DeviceProfileId deviceProfileId) { - return DeviceProfileUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(deviceProfileId.getId().getMostSignificantBits()) - .setIdLSB(deviceProfileId.getId().getLeastSignificantBits()).build(); - } - - @Override - public DeviceRpcCallMsg constructDeviceRpcCallMsg(UUID deviceId, JsonNode body) { - DeviceRpcCallMsg.Builder builder = constructDeviceRpcMsg(deviceId, body); - if (body.has("error") || body.has("response")) { - RpcResponseMsg.Builder responseBuilder = RpcResponseMsg.newBuilder(); - if (body.has("error")) { - responseBuilder.setError(body.get("error").asText()); - } else { - responseBuilder.setResponse(body.get("response").asText()); - } - builder.setResponseMsg(responseBuilder.build()); - } else { - RpcRequestMsg.Builder requestBuilder = RpcRequestMsg.newBuilder(); - requestBuilder.setMethod(body.get("method").asText()); - requestBuilder.setParams(body.get("params").asText()); - builder.setRequestMsg(requestBuilder.build()); - } - return builder.build(); - } - - private DeviceRpcCallMsg.Builder constructDeviceRpcMsg(UUID deviceId, JsonNode body) { - DeviceRpcCallMsg.Builder builder = DeviceRpcCallMsg.newBuilder() - .setDeviceIdMSB(deviceId.getMostSignificantBits()) - .setDeviceIdLSB(deviceId.getLeastSignificantBits()) - .setRequestId(body.get("requestId").asInt()); - if (body.get("oneway") != null) { - builder.setOneway(body.get("oneway").asBoolean()); - } - if (body.get("requestUUID") != null) { - UUID requestUUID = UUID.fromString(body.get("requestUUID").asText()); - builder.setRequestUuidMSB(requestUUID.getMostSignificantBits()) - .setRequestUuidLSB(requestUUID.getLeastSignificantBits()); - } - if (body.get("expirationTime") != null) { - builder.setExpirationTime(body.get("expirationTime").asLong()); - } - if (body.get("persisted") != null) { - builder.setPersisted(body.get("persisted").asBoolean()); - } - if (body.get("retries") != null) { - builder.setRetries(body.get("retries").asInt()); - } - if (body.get("additionalInfo") != null) { - builder.setAdditionalInfo(JacksonUtil.toString(body.get("additionalInfo"))); - } - if (body.get("serviceId") != null) { - builder.setServiceId(body.get("serviceId").asText()); - } - if (body.get("sessionId") != null) { - builder.setSessionId(body.get("sessionId").asText()); - } - return builder; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructor.java deleted file mode 100644 index 854d0c80a8..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructor.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.device; - -import com.fasterxml.jackson.databind.JsonNode; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; -import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -import java.util.UUID; - -public interface DeviceMsgConstructor extends MsgConstructor { - - DeviceUpdateMsg constructDeviceUpdatedMsg(UpdateMsgType msgType, Device device); - - DeviceUpdateMsg constructDeviceDeleteMsg(DeviceId deviceId); - - DeviceCredentialsUpdateMsg constructDeviceCredentialsUpdatedMsg(DeviceCredentials deviceCredentials); - - DeviceProfileUpdateMsg constructDeviceProfileUpdatedMsg(UpdateMsgType msgType, DeviceProfile deviceProfile); - - DeviceProfileUpdateMsg constructDeviceProfileDeleteMsg(DeviceProfileId deviceProfileId); - - DeviceRpcCallMsg constructDeviceRpcCallMsg(UUID deviceId, JsonNode body); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorFactory.java deleted file mode 100644 index 536e8db45b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.device; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class DeviceMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV1.java deleted file mode 100644 index 7b59bab3ba..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV1.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.device; - -import com.google.protobuf.ByteString; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.dao.resource.ImageService; -import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.nio.charset.StandardCharsets; - -@Component -@TbCoreComponent -public class DeviceMsgConstructorV1 extends BaseDeviceMsgConstructor { - - @Autowired - private ImageService imageService; - - @Override - public DeviceUpdateMsg constructDeviceUpdatedMsg(UpdateMsgType msgType, Device device) { - DeviceUpdateMsg.Builder builder = DeviceUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(device.getId().getId().getMostSignificantBits()) - .setIdLSB(device.getId().getId().getLeastSignificantBits()) - .setName(device.getName()) - .setType(device.getType()); - if (device.getLabel() != null) { - builder.setLabel(device.getLabel()); - } - if (device.getCustomerId() != null) { - builder.setCustomerIdMSB(device.getCustomerId().getId().getMostSignificantBits()); - builder.setCustomerIdLSB(device.getCustomerId().getId().getLeastSignificantBits()); - } - if (device.getDeviceProfileId() != null) { - builder.setDeviceProfileIdMSB(device.getDeviceProfileId().getId().getMostSignificantBits()); - builder.setDeviceProfileIdLSB(device.getDeviceProfileId().getId().getLeastSignificantBits()); - } - if (device.getAdditionalInfo() != null) { - builder.setAdditionalInfo(JacksonUtil.toString(device.getAdditionalInfo())); - } - if (device.getFirmwareId() != null) { - builder.setFirmwareIdMSB(device.getFirmwareId().getId().getMostSignificantBits()) - .setFirmwareIdLSB(device.getFirmwareId().getId().getLeastSignificantBits()); - } - if (device.getSoftwareId() != null) { - builder.setSoftwareIdMSB(device.getSoftwareId().getId().getMostSignificantBits()) - .setSoftwareIdLSB(device.getSoftwareId().getId().getLeastSignificantBits()); - } - if (device.getDeviceData() != null) { - builder.setDeviceDataBytes(ByteString.copyFrom(device.getDeviceDataBytes())); - } - return builder.build(); - } - - @Override - public DeviceCredentialsUpdateMsg constructDeviceCredentialsUpdatedMsg(DeviceCredentials deviceCredentials) { - DeviceCredentialsUpdateMsg.Builder builder = DeviceCredentialsUpdateMsg.newBuilder() - .setDeviceIdMSB(deviceCredentials.getDeviceId().getId().getMostSignificantBits()) - .setDeviceIdLSB(deviceCredentials.getDeviceId().getId().getLeastSignificantBits()); - if (deviceCredentials.getCredentialsType() != null) { - builder.setCredentialsType(deviceCredentials.getCredentialsType().name()) - .setCredentialsId(deviceCredentials.getCredentialsId()); - } - if (deviceCredentials.getCredentialsValue() != null) { - builder.setCredentialsValue(deviceCredentials.getCredentialsValue()); - } - return builder.build(); - } - - @Override - public DeviceProfileUpdateMsg constructDeviceProfileUpdatedMsg(UpdateMsgType msgType, DeviceProfile deviceProfile) { - deviceProfile = JacksonUtil.clone(deviceProfile); - imageService.inlineImageForEdge(deviceProfile); - DeviceProfileUpdateMsg.Builder builder = DeviceProfileUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(deviceProfile.getId().getId().getMostSignificantBits()) - .setIdLSB(deviceProfile.getId().getId().getLeastSignificantBits()) - .setName(deviceProfile.getName()) - .setDefault(deviceProfile.isDefault()) - .setType(deviceProfile.getType().name()) - .setProfileDataBytes(ByteString.copyFrom(deviceProfile.getProfileDataBytes())); - if (deviceProfile.getDefaultQueueName() != null) { - builder.setDefaultQueueName(deviceProfile.getDefaultQueueName()); - } - if (deviceProfile.getDescription() != null) { - builder.setDescription(deviceProfile.getDescription()); - } - if (deviceProfile.getTransportType() != null) { - builder.setTransportType(deviceProfile.getTransportType().name()); - } - if (deviceProfile.getProvisionType() != null) { - builder.setProvisionType(deviceProfile.getProvisionType().name()); - } - if (deviceProfile.getProvisionDeviceKey() != null) { - builder.setProvisionDeviceKey(deviceProfile.getProvisionDeviceKey()); - } - if (deviceProfile.getImage() != null) { - builder.setImage(ByteString.copyFrom(deviceProfile.getImage().getBytes(StandardCharsets.UTF_8))); - } - if (deviceProfile.getFirmwareId() != null) { - builder.setFirmwareIdMSB(deviceProfile.getFirmwareId().getId().getMostSignificantBits()) - .setFirmwareIdLSB(deviceProfile.getFirmwareId().getId().getLeastSignificantBits()); - } - if (deviceProfile.getSoftwareId() != null) { - builder.setSoftwareIdMSB(deviceProfile.getSoftwareId().getId().getMostSignificantBits()) - .setSoftwareIdLSB(deviceProfile.getSoftwareId().getId().getLeastSignificantBits()); - } - if (deviceProfile.getDefaultEdgeRuleChainId() != null) { - builder.setDefaultRuleChainIdMSB(deviceProfile.getDefaultEdgeRuleChainId().getId().getMostSignificantBits()) - .setDefaultRuleChainIdLSB(deviceProfile.getDefaultEdgeRuleChainId().getId().getLeastSignificantBits()); - } - if (deviceProfile.getDefaultDashboardId() != null) { - builder.setDefaultDashboardIdMSB(deviceProfile.getDefaultDashboardId().getId().getMostSignificantBits()) - .setDefaultDashboardIdLSB(deviceProfile.getDefaultDashboardId().getId().getLeastSignificantBits()); - } - return builder.build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV2.java deleted file mode 100644 index cf951e39a9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV2.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.device; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class DeviceMsgConstructorV2 extends BaseDeviceMsgConstructor { - - @Override - public DeviceUpdateMsg constructDeviceUpdatedMsg(UpdateMsgType msgType, Device device) { - return DeviceUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(device)) - .setIdMSB(device.getId().getId().getMostSignificantBits()) - .setIdLSB(device.getId().getId().getLeastSignificantBits()).build(); - } - - @Override - public DeviceCredentialsUpdateMsg constructDeviceCredentialsUpdatedMsg(DeviceCredentials deviceCredentials) { - return DeviceCredentialsUpdateMsg.newBuilder().setEntity(JacksonUtil.toString(deviceCredentials)).build(); - } - - @Override - public DeviceProfileUpdateMsg constructDeviceProfileUpdatedMsg(UpdateMsgType msgType, DeviceProfile deviceProfile) { - return DeviceProfileUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(deviceProfile)) - .setIdMSB(deviceProfile.getId().getId().getMostSignificantBits()) - .setIdLSB(deviceProfile.getId().getId().getLeastSignificantBits()).build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/edge/EdgeMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/edge/EdgeMsgConstructor.java deleted file mode 100644 index 4b45b0427f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/edge/EdgeMsgConstructor.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.edge; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class EdgeMsgConstructor { - - public EdgeConfiguration constructEdgeConfiguration(Edge edge) { - EdgeConfiguration.Builder builder = EdgeConfiguration.newBuilder() - .setEdgeIdMSB(edge.getId().getId().getMostSignificantBits()) - .setEdgeIdLSB(edge.getId().getId().getLeastSignificantBits()) - .setTenantIdMSB(edge.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(edge.getTenantId().getId().getLeastSignificantBits()) - .setName(edge.getName()) - .setType(edge.getType()) - .setRoutingKey(edge.getRoutingKey()) - .setSecret(edge.getSecret()) - .setAdditionalInfo(JacksonUtil.toString(edge.getAdditionalInfo())) - .setCloudType("CE"); - if (edge.getCustomerId() != null) { - builder.setCustomerIdMSB(edge.getCustomerId().getId().getMostSignificantBits()) - .setCustomerIdLSB(edge.getCustomerId().getId().getLeastSignificantBits()); - } - return builder.build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/BaseEntityViewMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/BaseEntityViewMsgConstructor.java deleted file mode 100644 index 7fa4a03c3e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/BaseEntityViewMsgConstructor.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.entityview; - -import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -public abstract class BaseEntityViewMsgConstructor implements EntityViewMsgConstructor { - - @Override - public EntityViewUpdateMsg constructEntityViewDeleteMsg(EntityViewId entityViewId) { - return EntityViewUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(entityViewId.getId().getMostSignificantBits()) - .setIdLSB(entityViewId.getId().getLeastSignificantBits()).build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructor.java deleted file mode 100644 index bec1d416d3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructor.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.entityview; - -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface EntityViewMsgConstructor extends MsgConstructor { - - EntityViewUpdateMsg constructEntityViewUpdatedMsg(UpdateMsgType msgType, EntityView entityView); - - EntityViewUpdateMsg constructEntityViewDeleteMsg(EntityViewId entityViewId); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorFactory.java deleted file mode 100644 index 2754622a67..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.entityview; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class EntityViewMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorV1.java deleted file mode 100644 index 1b3bba22c0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorV1.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.entityview; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.gen.edge.v1.EdgeEntityType; -import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class EntityViewMsgConstructorV1 extends BaseEntityViewMsgConstructor { - - @Override - public EntityViewUpdateMsg constructEntityViewUpdatedMsg(UpdateMsgType msgType, EntityView entityView) { - EdgeEntityType edgeEntityType = checkEntityType(entityView.getEntityId().getEntityType()); - EntityViewUpdateMsg.Builder builder = EntityViewUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(entityView.getId().getId().getMostSignificantBits()) - .setIdLSB(entityView.getId().getId().getLeastSignificantBits()) - .setName(entityView.getName()) - .setType(entityView.getType()) - .setEntityIdMSB(entityView.getEntityId().getId().getMostSignificantBits()) - .setEntityIdLSB(entityView.getEntityId().getId().getLeastSignificantBits()) - .setEntityType(edgeEntityType); - if (entityView.getCustomerId() != null) { - builder.setCustomerIdMSB(entityView.getCustomerId().getId().getMostSignificantBits()); - builder.setCustomerIdLSB(entityView.getCustomerId().getId().getLeastSignificantBits()); - } - if (entityView.getAdditionalInfo() != null) { - builder.setAdditionalInfo(JacksonUtil.toString(entityView.getAdditionalInfo())); - } - return builder.build(); - } - - private EdgeEntityType checkEntityType(EntityType entityType) { - return switch (entityType) { - case DEVICE -> EdgeEntityType.DEVICE; - case ASSET -> EdgeEntityType.ASSET; - default -> throw new RuntimeException("Unsupported entity type [" + entityType + "]"); - }; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorV2.java deleted file mode 100644 index 6db4818688..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/entityview/EntityViewMsgConstructorV2.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.entityview; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class EntityViewMsgConstructorV2 extends BaseEntityViewMsgConstructor { - - @Override - public EntityViewUpdateMsg constructEntityViewUpdatedMsg(UpdateMsgType msgType, EntityView entityView) { - return EntityViewUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(entityView)) - .setIdMSB(entityView.getId().getId().getMostSignificantBits()) - .setIdLSB(entityView.getId().getId().getLeastSignificantBits()).build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/notification/NotificationMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/notification/NotificationMsgConstructor.java deleted file mode 100644 index 9c8f5a37e6..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/notification/NotificationMsgConstructor.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.notification; - -import org.thingsboard.server.common.data.id.NotificationRuleId; -import org.thingsboard.server.common.data.id.NotificationTargetId; -import org.thingsboard.server.common.data.id.NotificationTemplateId; -import org.thingsboard.server.common.data.notification.rule.NotificationRule; -import org.thingsboard.server.common.data.notification.targets.NotificationTarget; -import org.thingsboard.server.common.data.notification.template.NotificationTemplate; -import org.thingsboard.server.gen.edge.v1.NotificationRuleUpdateMsg; -import org.thingsboard.server.gen.edge.v1.NotificationTargetUpdateMsg; -import org.thingsboard.server.gen.edge.v1.NotificationTemplateUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -public interface NotificationMsgConstructor { - - NotificationRuleUpdateMsg constructNotificationRuleUpdateMsg(UpdateMsgType msgType, NotificationRule notificationRule); - - NotificationRuleUpdateMsg constructNotificationRuleDeleteMsg(NotificationRuleId notificationRuleId); - - NotificationTargetUpdateMsg constructNotificationTargetUpdateMsg(UpdateMsgType msgType, NotificationTarget notificationTarget); - - NotificationTargetUpdateMsg constructNotificationTargetDeleteMsg(NotificationTargetId notificationTargetId); - - NotificationTemplateUpdateMsg constructNotificationTemplateUpdateMsg(UpdateMsgType msgType, NotificationTemplate notificationTemplate); - - NotificationTemplateUpdateMsg constructNotificationTemplateDeleteMsg(NotificationTemplateId notificationTemplateId); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/notification/NotificationMsgConstructorImpl.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/notification/NotificationMsgConstructorImpl.java deleted file mode 100644 index 8340bee693..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/notification/NotificationMsgConstructorImpl.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.notification; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.id.NotificationRuleId; -import org.thingsboard.server.common.data.id.NotificationTargetId; -import org.thingsboard.server.common.data.id.NotificationTemplateId; -import org.thingsboard.server.common.data.notification.rule.NotificationRule; -import org.thingsboard.server.common.data.notification.targets.NotificationTarget; -import org.thingsboard.server.common.data.notification.template.NotificationTemplate; -import org.thingsboard.server.gen.edge.v1.NotificationRuleUpdateMsg; -import org.thingsboard.server.gen.edge.v1.NotificationTargetUpdateMsg; -import org.thingsboard.server.gen.edge.v1.NotificationTemplateUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class NotificationMsgConstructorImpl implements NotificationMsgConstructor { - - @Override - public NotificationRuleUpdateMsg constructNotificationRuleUpdateMsg(UpdateMsgType msgType, NotificationRule notificationRule) { - return NotificationRuleUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(notificationRule)).build(); - } - - @Override - public NotificationRuleUpdateMsg constructNotificationRuleDeleteMsg(NotificationRuleId notificationRuleId) { - return NotificationRuleUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(notificationRuleId.getId().getMostSignificantBits()) - .setIdLSB(notificationRuleId.getId().getLeastSignificantBits()).build(); - } - - @Override - public NotificationTargetUpdateMsg constructNotificationTargetUpdateMsg(UpdateMsgType msgType, NotificationTarget notificationTarget) { - return NotificationTargetUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(notificationTarget)).build(); - } - - @Override - public NotificationTargetUpdateMsg constructNotificationTargetDeleteMsg(NotificationTargetId notificationTargetId) { - return NotificationTargetUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(notificationTargetId.getId().getMostSignificantBits()) - .setIdLSB(notificationTargetId.getId().getLeastSignificantBits()).build(); - } - - @Override - public NotificationTemplateUpdateMsg constructNotificationTemplateUpdateMsg(UpdateMsgType msgType, NotificationTemplate notificationTemplate) { - return NotificationTemplateUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(notificationTemplate)).build(); - } - - @Override - public NotificationTemplateUpdateMsg constructNotificationTemplateDeleteMsg(NotificationTemplateId notificationTemplateId) { - return NotificationTemplateUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(notificationTemplateId.getId().getMostSignificantBits()) - .setIdLSB(notificationTemplateId.getId().getLeastSignificantBits()).build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/oauth2/OAuth2MsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/oauth2/OAuth2MsgConstructor.java deleted file mode 100644 index 7ed25924b6..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/oauth2/OAuth2MsgConstructor.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.oauth2; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.domain.DomainInfo; -import org.thingsboard.server.common.data.id.DomainId; -import org.thingsboard.server.common.data.id.OAuth2ClientId; -import org.thingsboard.server.common.data.oauth2.OAuth2Client; -import org.thingsboard.server.gen.edge.v1.OAuth2ClientUpdateMsg; -import org.thingsboard.server.gen.edge.v1.OAuth2DomainUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class OAuth2MsgConstructor { - - public OAuth2ClientUpdateMsg constructOAuth2ClientUpdateMsg(UpdateMsgType msgType, OAuth2Client oAuth2Client) { - return OAuth2ClientUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(oAuth2Client)) - .setIdMSB(oAuth2Client.getId().getId().getMostSignificantBits()) - .setIdLSB(oAuth2Client.getId().getId().getLeastSignificantBits()).build(); - } - - public OAuth2ClientUpdateMsg constructOAuth2ClientDeleteMsg(OAuth2ClientId oAuth2ClientId) { - return OAuth2ClientUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(oAuth2ClientId.getId().getMostSignificantBits()) - .setIdLSB(oAuth2ClientId.getId().getLeastSignificantBits()).build(); - } - - public OAuth2DomainUpdateMsg constructOAuth2DomainUpdateMsg(UpdateMsgType msgType, DomainInfo domainInfo) { - return OAuth2DomainUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(domainInfo)) - .setIdMSB(domainInfo.getId().getId().getMostSignificantBits()) - .setIdLSB(domainInfo.getId().getId().getLeastSignificantBits()).build(); - } - - public OAuth2DomainUpdateMsg constructOAuth2DomainDeleteMsg(DomainId domainId) { - return OAuth2DomainUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(domainId.getId().getMostSignificantBits()) - .setIdLSB(domainId.getId().getLeastSignificantBits()) - .build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/BaseOtaPackageMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/BaseOtaPackageMsgConstructor.java deleted file mode 100644 index c06d6adc4b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/BaseOtaPackageMsgConstructor.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.ota; - -import org.thingsboard.server.common.data.id.OtaPackageId; -import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -public abstract class BaseOtaPackageMsgConstructor implements OtaPackageMsgConstructor { - - @Override - public OtaPackageUpdateMsg constructOtaPackageDeleteMsg(OtaPackageId otaPackageId) { - return OtaPackageUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(otaPackageId.getId().getMostSignificantBits()) - .setIdLSB(otaPackageId.getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructor.java deleted file mode 100644 index 247043eb20..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructor.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.ota; - -import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.common.data.id.OtaPackageId; -import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface OtaPackageMsgConstructor extends MsgConstructor { - - OtaPackageUpdateMsg constructOtaPackageUpdatedMsg(UpdateMsgType msgType, OtaPackage otaPackage); - - OtaPackageUpdateMsg constructOtaPackageDeleteMsg(OtaPackageId otaPackageId); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorFactory.java deleted file mode 100644 index 664ec1d60b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.ota; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class OtaPackageMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorV1.java deleted file mode 100644 index 3c743a4200..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorV1.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.ota; - -import com.google.protobuf.ByteString; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class OtaPackageMsgConstructorV1 extends BaseOtaPackageMsgConstructor { - - @Override - public OtaPackageUpdateMsg constructOtaPackageUpdatedMsg(UpdateMsgType msgType, OtaPackage otaPackage) { - OtaPackageUpdateMsg.Builder builder = OtaPackageUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(otaPackage.getId().getId().getMostSignificantBits()) - .setIdLSB(otaPackage.getId().getId().getLeastSignificantBits()) - .setType(otaPackage.getType().name()) - .setTitle(otaPackage.getTitle()) - .setVersion(otaPackage.getVersion()) - .setTag(otaPackage.getTag()); - - if (otaPackage.getDeviceProfileId() != null) { - builder.setDeviceProfileIdMSB(otaPackage.getDeviceProfileId().getId().getMostSignificantBits()) - .setDeviceProfileIdLSB(otaPackage.getDeviceProfileId().getId().getLeastSignificantBits()); - } - - if (otaPackage.getUrl() != null) { - builder.setUrl(otaPackage.getUrl()); - } - if (otaPackage.getAdditionalInfo() != null) { - builder.setAdditionalInfo(JacksonUtil.toString(otaPackage.getAdditionalInfo())); - } - if (otaPackage.getFileName() != null) { - builder.setFileName(otaPackage.getFileName()); - } - if (otaPackage.getContentType() != null) { - builder.setContentType(otaPackage.getContentType()); - } - if (otaPackage.getChecksumAlgorithm() != null) { - builder.setChecksumAlgorithm(otaPackage.getChecksumAlgorithm().name()); - } - if (otaPackage.getChecksum() != null) { - builder.setChecksum(otaPackage.getChecksum()); - } - if (otaPackage.getDataSize() != null) { - builder.setDataSize(otaPackage.getDataSize()); - } - if (otaPackage.getData() != null) { - builder.setData(ByteString.copyFrom(otaPackage.getData().array())); - } - return builder.build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorV2.java deleted file mode 100644 index c3c7eac9a4..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/ota/OtaPackageMsgConstructorV2.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.ota; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class OtaPackageMsgConstructorV2 extends BaseOtaPackageMsgConstructor { - - @Override - public OtaPackageUpdateMsg constructOtaPackageUpdatedMsg(UpdateMsgType msgType, OtaPackage otaPackage) { - return OtaPackageUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(otaPackage)) - .setIdMSB(otaPackage.getId().getId().getMostSignificantBits()) - .setIdLSB(otaPackage.getId().getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/BaseQueueMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/BaseQueueMsgConstructor.java deleted file mode 100644 index ce68ef8bcb..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/BaseQueueMsgConstructor.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.queue; - -import org.thingsboard.server.common.data.id.QueueId; -import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -public abstract class BaseQueueMsgConstructor implements QueueMsgConstructor { - - @Override - public QueueUpdateMsg constructQueueDeleteMsg(QueueId queueId) { - return QueueUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(queueId.getId().getMostSignificantBits()) - .setIdLSB(queueId.getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructor.java deleted file mode 100644 index 1aad30fb8f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructor.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.queue; - -import org.thingsboard.server.common.data.id.QueueId; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface QueueMsgConstructor extends MsgConstructor { - - QueueUpdateMsg constructQueueUpdatedMsg(UpdateMsgType msgType, Queue queue); - - QueueUpdateMsg constructQueueDeleteMsg(QueueId queueId); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorFactory.java deleted file mode 100644 index 5549a2fdba..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.queue; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class QueueMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorV1.java deleted file mode 100644 index 78cdc0f7e4..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorV1.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.queue; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.id.QueueId; -import org.thingsboard.server.common.data.queue.ProcessingStrategy; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.data.queue.SubmitStrategy; -import org.thingsboard.server.gen.edge.v1.ProcessingStrategyProto; -import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg; -import org.thingsboard.server.gen.edge.v1.SubmitStrategyProto; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class QueueMsgConstructorV1 extends BaseQueueMsgConstructor { - - @Override - public QueueUpdateMsg constructQueueUpdatedMsg(UpdateMsgType msgType, Queue queue) { - return QueueUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(queue.getId().getId().getMostSignificantBits()) - .setIdLSB(queue.getId().getId().getLeastSignificantBits()) - .setTenantIdMSB(queue.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(queue.getTenantId().getId().getLeastSignificantBits()) - .setName(queue.getName()) - .setTopic(queue.getTopic()) - .setPollInterval(queue.getPollInterval()) - .setPartitions(queue.getPartitions()) - .setConsumerPerPartition(queue.isConsumerPerPartition()) - .setPackProcessingTimeout(queue.getPackProcessingTimeout()) - .setSubmitStrategy(createSubmitStrategyProto(queue.getSubmitStrategy())) - .setProcessingStrategy(createProcessingStrategyProto(queue.getProcessingStrategy())).build(); - } - - private ProcessingStrategyProto createProcessingStrategyProto(ProcessingStrategy processingStrategy) { - return ProcessingStrategyProto.newBuilder() - .setType(processingStrategy.getType().name()) - .setRetries(processingStrategy.getRetries()) - .setFailurePercentage(processingStrategy.getFailurePercentage()) - .setPauseBetweenRetries(processingStrategy.getPauseBetweenRetries()) - .setMaxPauseBetweenRetries(processingStrategy.getMaxPauseBetweenRetries()) - .build(); - } - - private SubmitStrategyProto createSubmitStrategyProto(SubmitStrategy submitStrategy) { - return SubmitStrategyProto.newBuilder() - .setType(submitStrategy.getType().name()) - .setBatchSize(submitStrategy.getBatchSize()) - .build(); - } - - public QueueUpdateMsg constructQueueDeleteMsg(QueueId queueId) { - return QueueUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(queueId.getId().getMostSignificantBits()) - .setIdLSB(queueId.getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorV2.java deleted file mode 100644 index 9df94b957e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/queue/QueueMsgConstructorV2.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.queue; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class QueueMsgConstructorV2 extends BaseQueueMsgConstructor { - - @Override - public QueueUpdateMsg constructQueueUpdatedMsg(UpdateMsgType msgType, Queue queue) { - return QueueUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(queue)) - .setIdMSB(queue.getId().getId().getMostSignificantBits()) - .setIdLSB(queue.getId().getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructor.java deleted file mode 100644 index 4dc1cd6777..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructor.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.relation; - -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface RelationMsgConstructor extends MsgConstructor { - - RelationUpdateMsg constructRelationUpdatedMsg(UpdateMsgType msgType, EntityRelation entityRelation); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorFactory.java deleted file mode 100644 index dd1642c6a0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.relation; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class RelationMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorV1.java deleted file mode 100644 index f5b9ae59f3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorV1.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.relation; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class RelationMsgConstructorV1 implements RelationMsgConstructor { - - @Override - public RelationUpdateMsg constructRelationUpdatedMsg(UpdateMsgType msgType, EntityRelation entityRelation) { - RelationUpdateMsg.Builder builder = RelationUpdateMsg.newBuilder() - .setMsgType(msgType) - .setFromIdMSB(entityRelation.getFrom().getId().getMostSignificantBits()) - .setFromIdLSB(entityRelation.getFrom().getId().getLeastSignificantBits()) - .setFromEntityType(entityRelation.getFrom().getEntityType().name()) - .setToIdMSB(entityRelation.getTo().getId().getMostSignificantBits()) - .setToIdLSB(entityRelation.getTo().getId().getLeastSignificantBits()) - .setToEntityType(entityRelation.getTo().getEntityType().name()) - .setType(entityRelation.getType()); - if (entityRelation.getAdditionalInfo() != null) { - builder.setAdditionalInfo(JacksonUtil.toString(entityRelation.getAdditionalInfo())); - } - if (entityRelation.getTypeGroup() != null) { - builder.setTypeGroup(entityRelation.getTypeGroup().name()); - } - return builder.build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorV2.java deleted file mode 100644 index 9a52b0b29b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/relation/RelationMsgConstructorV2.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.relation; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class RelationMsgConstructorV2 implements RelationMsgConstructor { - - - @Override - public RelationUpdateMsg constructRelationUpdatedMsg(UpdateMsgType msgType, EntityRelation entityRelation) { - return RelationUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(entityRelation)).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/BaseResourceMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/BaseResourceMsgConstructor.java deleted file mode 100644 index 0a9d34fb85..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/BaseResourceMsgConstructor.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.resource; - -import org.thingsboard.server.common.data.id.TbResourceId; -import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -public abstract class BaseResourceMsgConstructor implements ResourceMsgConstructor { - - @Override - public ResourceUpdateMsg constructResourceDeleteMsg(TbResourceId tbResourceId) { - return ResourceUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(tbResourceId.getId().getMostSignificantBits()) - .setIdLSB(tbResourceId.getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructor.java deleted file mode 100644 index 9b002237fb..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructor.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.resource; - -import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.id.TbResourceId; -import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface ResourceMsgConstructor extends MsgConstructor { - - ResourceUpdateMsg constructResourceUpdatedMsg(UpdateMsgType msgType, TbResource tbResource); - - ResourceUpdateMsg constructResourceDeleteMsg(TbResourceId tbResourceId); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorFactory.java deleted file mode 100644 index e2e3970b41..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.resource; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class ResourceMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV1.java deleted file mode 100644 index e1cf75ea56..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV1.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.resource; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.ResourceType; -import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class ResourceMsgConstructorV1 extends BaseResourceMsgConstructor { - - @Override - public ResourceUpdateMsg constructResourceUpdatedMsg(UpdateMsgType msgType, TbResource tbResource) { - if (ResourceType.IMAGE.equals(tbResource.getResourceType())) { - // Exclude support for a recently added resource type when dealing with older Edges - // to maintain compatibility and avoid potential issues. - return null; - } - ResourceUpdateMsg.Builder builder = ResourceUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(tbResource.getId().getId().getMostSignificantBits()) - .setIdLSB(tbResource.getId().getId().getLeastSignificantBits()) - .setTitle(tbResource.getTitle()) - .setResourceKey(tbResource.getResourceKey()) - .setResourceType(tbResource.getResourceType().name()) - .setFileName(tbResource.getFileName()); - if (tbResource.getData() != null) { - builder.setData(tbResource.getEncodedData()); - } - if (tbResource.getEtag() != null) { - builder.setEtag(tbResource.getEtag()); - } - if (TenantId.SYS_TENANT_ID.equals(tbResource.getTenantId())) { - builder.setIsSystem(true); - } - return builder.build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV2.java deleted file mode 100644 index 85aaa61def..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV2.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.resource; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class ResourceMsgConstructorV2 extends BaseResourceMsgConstructor { - - @Override - public ResourceUpdateMsg constructResourceUpdatedMsg(UpdateMsgType msgType, TbResource tbResource) { - return ResourceUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(tbResource)) - .setIdMSB(tbResource.getId().getId().getMostSignificantBits()) - .setIdLSB(tbResource.getId().getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/BaseRuleChainMetadataConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/BaseRuleChainMetadataConstructor.java deleted file mode 100644 index efa541ca54..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/BaseRuleChainMetadataConstructor.java +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.NodeConnectionInfo; -import org.thingsboard.server.common.data.rule.RuleChainConnectionInfo; -import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.NodeConnectionInfoProto; -import org.thingsboard.server.gen.edge.v1.RuleChainConnectionInfoProto; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; -import org.thingsboard.server.gen.edge.v1.RuleNodeProto; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -import java.util.ArrayList; -import java.util.List; -import java.util.NavigableSet; - -@Slf4j -@AllArgsConstructor -public abstract class BaseRuleChainMetadataConstructor implements RuleChainMetadataConstructor { - - @Override - public RuleChainMetadataUpdateMsg constructRuleChainMetadataUpdatedMsg(TenantId tenantId, - UpdateMsgType msgType, - RuleChainMetaData ruleChainMetaData, - EdgeVersion edgeVersion) { - RuleChainMetadataUpdateMsg.Builder builder = RuleChainMetadataUpdateMsg.newBuilder(); - constructRuleChainMetadataUpdatedMsg(tenantId, builder, ruleChainMetaData); - builder.setMsgType(msgType); - return builder.build(); - } - - protected abstract void constructRuleChainMetadataUpdatedMsg(TenantId tenantId, - RuleChainMetadataUpdateMsg.Builder builder, - RuleChainMetaData ruleChainMetaData); - - protected List constructConnections(List connections) { - List result = new ArrayList<>(); - if (connections != null && !connections.isEmpty()) { - for (NodeConnectionInfo connection : connections) { - result.add(constructConnection(connection)); - } - } - return result; - } - - private NodeConnectionInfoProto constructConnection(NodeConnectionInfo connection) { - return NodeConnectionInfoProto.newBuilder() - .setFromIndex(connection.getFromIndex()) - .setToIndex(connection.getToIndex()) - .setType(connection.getType()) - .build(); - } - - protected List constructNodes(List nodes) { - List result = new ArrayList<>(); - if (nodes != null && !nodes.isEmpty()) { - for (RuleNode node : nodes) { - result.add(constructNode(node)); - } - } - return result; - } - - private RuleNodeProto constructNode(RuleNode node) { - return RuleNodeProto.newBuilder() - .setIdMSB(node.getId().getId().getMostSignificantBits()) - .setIdLSB(node.getId().getId().getLeastSignificantBits()) - .setType(node.getType()) - .setName(node.getName()) - .setDebugSettings(JacksonUtil.toString(node.getDebugSettings())) - .setConfiguration(JacksonUtil.toString(node.getConfiguration())) - .setAdditionalInfo(JacksonUtil.toString(node.getAdditionalInfo())) - .setSingletonMode(node.isSingletonMode()) - .setConfigurationVersion(node.getConfigurationVersion()) - .build(); - } - - protected List constructRuleChainConnections(List ruleChainConnections, - NavigableSet removedNodeIndexes) { - List result = new ArrayList<>(); - if (ruleChainConnections != null && !ruleChainConnections.isEmpty()) { - for (RuleChainConnectionInfo ruleChainConnectionInfo : ruleChainConnections) { - if (!removedNodeIndexes.isEmpty()) { // 3_3_0 only - int fromIndex = ruleChainConnectionInfo.getFromIndex(); - // decrease index because of removed nodes - for (Integer removedIndex : removedNodeIndexes) { - if (fromIndex > removedIndex) { - fromIndex = fromIndex - 1; - } - } - ruleChainConnectionInfo.setFromIndex(fromIndex); - ObjectNode additionalInfo = (ObjectNode) ruleChainConnectionInfo.getAdditionalInfo(); - if (additionalInfo.get("ruleChainNodeId") == null) { - additionalInfo.put("ruleChainNodeId", "rule-chain-node-UNDEFINED"); - } - } - result.add(constructRuleChainConnection(ruleChainConnectionInfo)); - } - } - return result; - } - - private RuleChainConnectionInfoProto constructRuleChainConnection(RuleChainConnectionInfo ruleChainConnectionInfo) { - return RuleChainConnectionInfoProto.newBuilder() - .setFromIndex(ruleChainConnectionInfo.getFromIndex()) - .setTargetRuleChainIdMSB(ruleChainConnectionInfo.getTargetRuleChainId().getId().getMostSignificantBits()) - .setTargetRuleChainIdLSB(ruleChainConnectionInfo.getTargetRuleChainId().getId().getLeastSignificantBits()) - .setType(ruleChainConnectionInfo.getType()) - .setAdditionalInfo(JacksonUtil.toString(ruleChainConnectionInfo.getAdditionalInfo())) - .build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/BaseRuleChainMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/BaseRuleChainMsgConstructor.java deleted file mode 100644 index 89ac4123a4..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/BaseRuleChainMsgConstructor.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; -import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -public abstract class BaseRuleChainMsgConstructor implements RuleChainMsgConstructor { - - @Override - public RuleChainUpdateMsg constructRuleChainDeleteMsg(RuleChainId ruleChainId) { - return RuleChainUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(ruleChainId.getId().getMostSignificantBits()) - .setIdLSB(ruleChainId.getId().getLeastSignificantBits()).build(); - } - - @Override - public RuleChainMetadataUpdateMsg constructRuleChainMetadataUpdatedMsg(TenantId tenantId, - UpdateMsgType msgType, - RuleChainMetaData ruleChainMetaData, - EdgeVersion edgeVersion) { - RuleChainMetadataConstructor ruleChainMetadataConstructor = RuleChainMetadataConstructorFactory.getByEdgeVersion(edgeVersion); - return ruleChainMetadataConstructor.constructRuleChainMetadataUpdatedMsg(tenantId, msgType, ruleChainMetaData, edgeVersion); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructor.java deleted file mode 100644 index 43b103cb8a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructor.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; - -public interface RuleChainMetadataConstructor { - - RuleChainMetadataUpdateMsg constructRuleChainMetadataUpdatedMsg(TenantId tenantId, - UpdateMsgType msgType, - RuleChainMetaData ruleChainMetaData, - EdgeVersion edgeVersion); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorFactory.java deleted file mode 100644 index 22c36e47a6..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorFactory.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import org.thingsboard.server.gen.edge.v1.EdgeVersion; - -public final class RuleChainMetadataConstructorFactory { - - public static RuleChainMetadataConstructor getByEdgeVersion(EdgeVersion edgeVersion) { - switch (edgeVersion) { - case V_3_3_0: - return new RuleChainMetadataConstructorV330(); - case V_3_3_3: - case V_3_4_0: - case V_3_6_0: - case V_3_6_1: - return new RuleChainMetadataConstructorV340(); - case V_3_6_2: - default: - return new RuleChainMetadataConstructorV362(); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV330.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV330.java deleted file mode 100644 index fe24a8267a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV330.java +++ /dev/null @@ -1,166 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.rule.engine.flow.TbRuleChainInputNode; -import org.thingsboard.rule.engine.flow.TbRuleChainInputNodeConfiguration; -import org.thingsboard.rule.engine.flow.TbRuleChainOutputNode; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.NodeConnectionInfo; -import org.thingsboard.server.common.data.rule.RuleChainConnectionInfo; -import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; - -import java.util.ArrayList; -import java.util.List; -import java.util.NavigableSet; -import java.util.TreeSet; -import java.util.UUID; -import java.util.stream.Collectors; - -@Slf4j -public class RuleChainMetadataConstructorV330 extends BaseRuleChainMetadataConstructor { - - private static final String RULE_CHAIN_INPUT_NODE = TbRuleChainInputNode.class.getName(); - private static final String TB_RULE_CHAIN_OUTPUT_NODE = TbRuleChainOutputNode.class.getName(); - - @Override - protected void constructRuleChainMetadataUpdatedMsg(TenantId tenantId, - RuleChainMetadataUpdateMsg.Builder builder, - RuleChainMetaData ruleChainMetaData) { - builder.setRuleChainIdMSB(ruleChainMetaData.getRuleChainId().getId().getMostSignificantBits()) - .setRuleChainIdLSB(ruleChainMetaData.getRuleChainId().getId().getLeastSignificantBits()); - List supportedNodes = filterNodes(ruleChainMetaData.getNodes()); - - NavigableSet removedNodeIndexes = getRemovedNodeIndexes(ruleChainMetaData.getNodes(), ruleChainMetaData.getConnections()); - List connections = filterConnections(ruleChainMetaData.getNodes(), ruleChainMetaData.getConnections(), removedNodeIndexes); - - List ruleChainConnections = new ArrayList<>(); - if (ruleChainMetaData.getRuleChainConnections() != null) { - ruleChainConnections.addAll(ruleChainMetaData.getRuleChainConnections()); - } - ruleChainConnections.addAll(addRuleChainConnections(ruleChainMetaData.getNodes(), ruleChainMetaData.getConnections())); - builder.addAllNodes(constructNodes(supportedNodes)) - .addAllConnections(constructConnections(connections)) - .addAllRuleChainConnections(constructRuleChainConnections(ruleChainConnections, removedNodeIndexes)); - if (ruleChainMetaData.getFirstNodeIndex() != null) { - Integer firstNodeIndex = ruleChainMetaData.getFirstNodeIndex(); - // decrease index because of removed nodes - for (Integer removedIndex : removedNodeIndexes) { - if (firstNodeIndex > removedIndex) { - firstNodeIndex = firstNodeIndex - 1; - } - } - builder.setFirstNodeIndex(firstNodeIndex); - } else { - builder.setFirstNodeIndex(-1); - } - } - - private NavigableSet getRemovedNodeIndexes(List nodes, List connections) { - TreeSet removedIndexes = new TreeSet<>(); - for (NodeConnectionInfo connection : connections) { - for (int i = 0; i < nodes.size(); i++) { - RuleNode node = nodes.get(i); - if (node.getType().equalsIgnoreCase(RULE_CHAIN_INPUT_NODE) - || node.getType().equalsIgnoreCase(TB_RULE_CHAIN_OUTPUT_NODE)) { - if (connection.getFromIndex() == i || connection.getToIndex() == i) { - removedIndexes.add(i); - } - } - } - } - return removedIndexes.descendingSet(); - } - - private List filterConnections(List nodes, - List connections, - NavigableSet removedNodeIndexes) { - List result = new ArrayList<>(); - if (connections != null) { - result = connections.stream().filter(conn -> { - for (int i = 0; i < nodes.size(); i++) { - RuleNode node = nodes.get(i); - if (node.getType().equalsIgnoreCase(RULE_CHAIN_INPUT_NODE) - || node.getType().equalsIgnoreCase(TB_RULE_CHAIN_OUTPUT_NODE)) { - if (conn.getFromIndex() == i || conn.getToIndex() == i) { - return false; - } - } - } - return true; - }).map(conn -> { - NodeConnectionInfo newConn = new NodeConnectionInfo(); - newConn.setFromIndex(conn.getFromIndex()); - newConn.setToIndex(conn.getToIndex()); - newConn.setType(conn.getType()); - return newConn; - }).collect(Collectors.toList()); - } - - // decrease index because of removed nodes - for (Integer removedIndex : removedNodeIndexes) { - for (NodeConnectionInfo newConn : result) { - if (newConn.getToIndex() > removedIndex) { - newConn.setToIndex(newConn.getToIndex() - 1); - } - if (newConn.getFromIndex() > removedIndex) { - newConn.setFromIndex(newConn.getFromIndex() - 1); - } - } - } - - return result; - } - - private List filterNodes(List nodes) { - List result = new ArrayList<>(); - for (RuleNode node : nodes) { - if (RULE_CHAIN_INPUT_NODE.equals(node.getType()) - || TB_RULE_CHAIN_OUTPUT_NODE.equals(node.getType())) { - log.trace("Skipping not supported rule node {}", node); - } else { - result.add(node); - } - } - return result; - } - - private List addRuleChainConnections(List nodes, List connections) { - List result = new ArrayList<>(); - for (int i = 0; i < nodes.size(); i++) { - RuleNode node = nodes.get(i); - if (node.getType().equalsIgnoreCase(RULE_CHAIN_INPUT_NODE)) { - for (NodeConnectionInfo connection : connections) { - if (connection.getToIndex() == i) { - RuleChainConnectionInfo e = new RuleChainConnectionInfo(); - e.setFromIndex(connection.getFromIndex()); - TbRuleChainInputNodeConfiguration configuration = JacksonUtil.treeToValue(node.getConfiguration(), TbRuleChainInputNodeConfiguration.class); - e.setTargetRuleChainId(new RuleChainId(UUID.fromString(configuration.getRuleChainId()))); - e.setAdditionalInfo(node.getAdditionalInfo()); - e.setType(connection.getType()); - result.add(e); - } - } - } - } - return result; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV340.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV340.java deleted file mode 100644 index 087f3563b1..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV340.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; - -import java.util.TreeSet; - -@Slf4j -public class RuleChainMetadataConstructorV340 extends BaseRuleChainMetadataConstructor { - - @Override - protected void constructRuleChainMetadataUpdatedMsg(TenantId tenantId, - RuleChainMetadataUpdateMsg.Builder builder, - RuleChainMetaData ruleChainMetaData) { - builder.setRuleChainIdMSB(ruleChainMetaData.getRuleChainId().getId().getMostSignificantBits()) - .setRuleChainIdLSB(ruleChainMetaData.getRuleChainId().getId().getLeastSignificantBits()); - builder.addAllNodes(constructNodes(ruleChainMetaData.getNodes())) - .addAllConnections(constructConnections(ruleChainMetaData.getConnections())) - .addAllRuleChainConnections(constructRuleChainConnections(ruleChainMetaData.getRuleChainConnections(), new TreeSet<>())); - if (ruleChainMetaData.getFirstNodeIndex() != null) { - builder.setFirstNodeIndex(ruleChainMetaData.getFirstNodeIndex()); - } else { - builder.setFirstNodeIndex(-1); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV362.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV362.java deleted file mode 100644 index a62a275672..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMetadataConstructorV362.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; - -public class RuleChainMetadataConstructorV362 extends BaseRuleChainMetadataConstructor { - - @Override - protected void constructRuleChainMetadataUpdatedMsg(TenantId tenantId, RuleChainMetadataUpdateMsg.Builder builder, RuleChainMetaData ruleChainMetaData) { - builder.setEntity(JacksonUtil.toString(ruleChainMetaData)); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructor.java deleted file mode 100644 index 89cf21a08c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructor.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; -import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface RuleChainMsgConstructor extends MsgConstructor { - - RuleChainUpdateMsg constructRuleChainUpdatedMsg(UpdateMsgType msgType, RuleChain ruleChain, boolean isRoot); - - RuleChainUpdateMsg constructRuleChainDeleteMsg(RuleChainId ruleChainId); - - RuleChainMetadataUpdateMsg constructRuleChainMetadataUpdatedMsg(TenantId tenantId, - UpdateMsgType msgType, - RuleChainMetaData ruleChainMetaData, - EdgeVersion edgeVersion); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorFactory.java deleted file mode 100644 index cb7b116ea1..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class RuleChainMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorV1.java deleted file mode 100644 index c4e8871187..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorV1.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class RuleChainMsgConstructorV1 extends BaseRuleChainMsgConstructor { - - @Override - public RuleChainUpdateMsg constructRuleChainUpdatedMsg(UpdateMsgType msgType, RuleChain ruleChain, boolean isRoot) { - RuleChainUpdateMsg.Builder builder = RuleChainUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(ruleChain.getId().getId().getMostSignificantBits()) - .setIdLSB(ruleChain.getId().getId().getLeastSignificantBits()) - .setName(ruleChain.getName()) - .setRoot(isRoot) - .setDebugMode(ruleChain.isDebugMode()) - .setConfiguration(JacksonUtil.toString(ruleChain.getConfiguration())); - if (ruleChain.getFirstRuleNodeId() != null) { - builder.setFirstRuleNodeIdMSB(ruleChain.getFirstRuleNodeId().getId().getMostSignificantBits()) - .setFirstRuleNodeIdLSB(ruleChain.getFirstRuleNodeId().getId().getLeastSignificantBits()); - } - return builder.build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorV2.java deleted file mode 100644 index fb03067ca0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/rule/RuleChainMsgConstructorV2.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.rule; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class RuleChainMsgConstructorV2 extends BaseRuleChainMsgConstructor { - - @Override - public RuleChainUpdateMsg constructRuleChainUpdatedMsg(UpdateMsgType msgType, RuleChain ruleChain, boolean isRoot) { - boolean isTemplateRoot = ruleChain.isRoot(); - ruleChain.setRoot(isRoot); - RuleChainUpdateMsg result = RuleChainUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(ruleChain)) - .setIdMSB(ruleChain.getId().getId().getMostSignificantBits()) - .setIdLSB(ruleChain.getId().getId().getLeastSignificantBits()).build(); - ruleChain.setRoot(isTemplateRoot); - return result; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructor.java deleted file mode 100644 index a5fa5f3808..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructor.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.settings; - -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.gen.edge.v1.AdminSettingsUpdateMsg; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface AdminSettingsMsgConstructor extends MsgConstructor { - - AdminSettingsUpdateMsg constructAdminSettingsUpdateMsg(AdminSettings adminSettings); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorFactory.java deleted file mode 100644 index 472e582b52..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.settings; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class AdminSettingsMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorV1.java deleted file mode 100644 index 1f3942542d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorV1.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.settings; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.AdminSettingsUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class AdminSettingsMsgConstructorV1 implements AdminSettingsMsgConstructor { - - @Override - public AdminSettingsUpdateMsg constructAdminSettingsUpdateMsg(AdminSettings adminSettings) { - AdminSettingsUpdateMsg.Builder builder = AdminSettingsUpdateMsg.newBuilder() - .setKey(adminSettings.getKey()) - .setJsonValue(JacksonUtil.toString(adminSettings.getJsonValue())); - if (TenantId.SYS_TENANT_ID.equals(adminSettings.getTenantId())) { - builder.setIsSystem(true); - } - return builder.build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorV2.java deleted file mode 100644 index d3cd44370b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/settings/AdminSettingsMsgConstructorV2.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.settings; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.gen.edge.v1.AdminSettingsUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class AdminSettingsMsgConstructorV2 implements AdminSettingsMsgConstructor { - - @Override - public AdminSettingsUpdateMsg constructAdminSettingsUpdateMsg(AdminSettings adminSettings) { - return AdminSettingsUpdateMsg.newBuilder().setEntity(JacksonUtil.toString(adminSettings)).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/telemetry/EntityDataMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/telemetry/EntityDataMsgConstructor.java deleted file mode 100644 index 6493f97f57..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/telemetry/EntityDataMsgConstructor.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.telemetry; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; -import com.google.gson.reflect.TypeToken; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.adaptor.JsonConverter; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.AttributeDeleteMsg; -import org.thingsboard.server.gen.edge.v1.EntityDataProto; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.List; - -@Slf4j -@Component -@TbCoreComponent -public class EntityDataMsgConstructor { - - public EntityDataProto constructEntityDataMsg(TenantId tenantId, EntityId entityId, EdgeEventActionType actionType, JsonElement entityData) { - EntityDataProto.Builder builder = EntityDataProto.newBuilder() - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) - .setEntityType(entityId.getEntityType().name()); - switch (actionType) { - case TIMESERIES_UPDATED: - try { - JsonObject data = entityData.getAsJsonObject(); - long ts; - if (data.get("ts") != null && !data.get("ts").isJsonNull()) { - ts = data.getAsJsonPrimitive("ts").getAsLong(); - } else { - ts = System.currentTimeMillis(); - } - builder.setPostTelemetryMsg(JsonConverter.convertToTelemetryProto(data.getAsJsonObject("data"), ts)); - } catch (Exception e) { - log.warn("[{}][{}] Can't convert to telemetry proto, entityData [{}]", tenantId, entityId, entityData, e); - } - break; - case ATTRIBUTES_UPDATED: - try { - JsonObject data = entityData.getAsJsonObject(); - TransportProtos.PostAttributeMsg attributesUpdatedMsg = JsonConverter.convertToAttributesProto(data.getAsJsonObject("kv")); - builder.setAttributesUpdatedMsg(attributesUpdatedMsg); - builder.setPostAttributeScope(getScopeOfDefault(data)); - } catch (Exception e) { - log.warn("[{}][{}] Can't convert to AttributesUpdatedMsg proto, entityData [{}]", tenantId, entityId, entityData, e); - } - break; - case POST_ATTRIBUTES: - try { - JsonObject data = entityData.getAsJsonObject(); - TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(data.getAsJsonObject("kv")); - builder.setPostAttributesMsg(postAttributesMsg); - builder.setPostAttributeScope(getScopeOfDefault(data)); - } catch (Exception e) { - log.warn("[{}][{}] Can't convert to PostAttributesMsg, entityData [{}]", tenantId, entityId, entityData, e); - } - break; - case ATTRIBUTES_DELETED: - try { - AttributeDeleteMsg.Builder attributeDeleteMsg = AttributeDeleteMsg.newBuilder(); - attributeDeleteMsg.setScope(entityData.getAsJsonObject().getAsJsonPrimitive("scope").getAsString()); - JsonArray jsonArray = entityData.getAsJsonObject().getAsJsonArray("keys"); - List keys = new Gson().fromJson(jsonArray.toString(), new TypeToken<>(){}.getType()); - attributeDeleteMsg.addAllAttributeNames(keys); - attributeDeleteMsg.build(); - builder.setAttributeDeleteMsg(attributeDeleteMsg); - } catch (Exception e) { - log.warn("[{}][{}] Can't convert to AttributeDeleteMsg proto, entityData [{}]", tenantId, entityId, entityData, e); - } - break; - } - return builder.build(); - } - - private String getScopeOfDefault(JsonObject data) { - JsonPrimitive scope = data.getAsJsonPrimitive("scope"); - String result = DataConstants.SERVER_SCOPE; - if (scope != null && StringUtils.isNotBlank(scope.getAsString())) { - result = scope.getAsString(); - } - return result; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructor.java deleted file mode 100644 index a1299b3f97..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructor.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.tenant; - -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface TenantMsgConstructor extends MsgConstructor { - - TenantUpdateMsg constructTenantUpdateMsg(UpdateMsgType msgType, Tenant tenant); - - TenantProfileUpdateMsg constructTenantProfileUpdateMsg(UpdateMsgType msgType, TenantProfile tenantProfile, EdgeVersion edgeVersion); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorFactory.java deleted file mode 100644 index 09441d8e91..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.tenant; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class TenantMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV1.java deleted file mode 100644 index a9accf572d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV1.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.tenant; - -import com.google.protobuf.ByteString; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.utils.EdgeVersionUtils; - -@Component -@TbCoreComponent -public class TenantMsgConstructorV1 implements TenantMsgConstructor { - - @Override - public TenantUpdateMsg constructTenantUpdateMsg(UpdateMsgType msgType, Tenant tenant) { - TenantUpdateMsg.Builder builder = TenantUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(tenant.getId().getId().getMostSignificantBits()) - .setIdLSB(tenant.getId().getId().getLeastSignificantBits()) - .setTitle(tenant.getTitle()) - .setProfileIdMSB(tenant.getTenantProfileId().getId().getMostSignificantBits()) - .setProfileIdLSB(tenant.getTenantProfileId().getId().getLeastSignificantBits()) - .setRegion(tenant.getRegion()); - if (tenant.getCountry() != null) { - builder.setCountry(tenant.getCountry()); - } - if (tenant.getState() != null) { - builder.setState(tenant.getState()); - } - if (tenant.getCity() != null) { - builder.setCity(tenant.getCity()); - } - if (tenant.getAddress() != null) { - builder.setAddress(tenant.getAddress()); - } - if (tenant.getAddress2() != null) { - builder.setAddress2(tenant.getAddress2()); - } - if (tenant.getZip() != null) { - builder.setZip(tenant.getZip()); - } - if (tenant.getPhone() != null) { - builder.setPhone(tenant.getPhone()); - } - if (tenant.getEmail() != null) { - builder.setEmail(tenant.getEmail()); - } - if (tenant.getAdditionalInfo() != null) { - builder.setAdditionalInfo(JacksonUtil.toString(tenant.getAdditionalInfo())); - } - return builder.build(); - } - - @Override - public TenantProfileUpdateMsg constructTenantProfileUpdateMsg(UpdateMsgType msgType, TenantProfile tenantProfile, EdgeVersion edgeVersion) { - tenantProfile = JacksonUtil.clone(tenantProfile); - // clear all config - var tenantProfileData = tenantProfile.getProfileData(); - var configuration = tenantProfile.getDefaultProfileConfiguration(); - configuration.setRpcTtlDays(0); - configuration.setMaxJSExecutions(0); - configuration.setMaxREExecutions(0); - configuration.setMaxDPStorageDays(0); - configuration.setMaxTbelExecutions(0); - configuration.setQueueStatsTtlDays(0); - configuration.setMaxTransportMessages(0); - configuration.setDefaultStorageTtlDays(0); - configuration.setMaxTransportDataPoints(0); - configuration.setRuleEngineExceptionsTtlDays(0); - configuration.setMaxRuleNodeExecutionsPerMessage(0); - configuration.setMaxDebugModeDurationMinutes(0); - tenantProfileData.setConfiguration(configuration); - tenantProfile.setProfileData(tenantProfileData); - - ByteString profileData = EdgeVersionUtils.isEdgeVersionOlderThan(edgeVersion, EdgeVersion.V_3_6_2) ? - ByteString.empty() : ByteString.copyFrom(tenantProfile.getProfileDataBytes()); - TenantProfileUpdateMsg.Builder builder = TenantProfileUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(tenantProfile.getId().getId().getMostSignificantBits()) - .setIdLSB(tenantProfile.getId().getId().getLeastSignificantBits()) - .setName(tenantProfile.getName()) - .setDefault(tenantProfile.isDefault()) - .setIsolatedRuleChain(tenantProfile.isIsolatedTbRuleEngine()) - .setProfileDataBytes(profileData); - if (tenantProfile.getDescription() != null) { - builder.setDescription(tenantProfile.getDescription()); - } - return builder.build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV2.java deleted file mode 100644 index ee0c6a0d72..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV2.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.tenant; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class TenantMsgConstructorV2 implements TenantMsgConstructor { - - @Override - public TenantUpdateMsg constructTenantUpdateMsg(UpdateMsgType msgType, Tenant tenant) { - return TenantUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(tenant)).build(); - } - - @Override - public TenantProfileUpdateMsg constructTenantProfileUpdateMsg(UpdateMsgType msgType, TenantProfile tenantProfile, EdgeVersion edgeVersion) { - return TenantProfileUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(tenantProfile)).build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/BaseUserMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/BaseUserMsgConstructor.java deleted file mode 100644 index ab2e2dec14..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/BaseUserMsgConstructor.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.user; - -import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; - -public abstract class BaseUserMsgConstructor implements UserMsgConstructor { - - @Override - public UserUpdateMsg constructUserDeleteMsg(UserId userId) { - return UserUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(userId.getId().getMostSignificantBits()) - .setIdLSB(userId.getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructor.java deleted file mode 100644 index f0fb9bd8a5..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructor.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.user; - -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -public interface UserMsgConstructor extends MsgConstructor { - - UserUpdateMsg constructUserUpdatedMsg(UpdateMsgType msgType, User user); - - UserUpdateMsg constructUserDeleteMsg(UserId userId); - - UserCredentialsUpdateMsg constructUserCredentialsUpdatedMsg(UserCredentials userCredentials); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorFactory.java deleted file mode 100644 index f2e42a1dcc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.user; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class UserMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorV1.java deleted file mode 100644 index 309de42b1e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorV1.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.user; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class UserMsgConstructorV1 extends BaseUserMsgConstructor { - - @Override - public UserUpdateMsg constructUserUpdatedMsg(UpdateMsgType msgType, User user) { - UserUpdateMsg.Builder builder = UserUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(user.getId().getId().getMostSignificantBits()) - .setIdLSB(user.getId().getId().getLeastSignificantBits()) - .setEmail(user.getEmail()) - .setAuthority(user.getAuthority().name()); - if (user.getCustomerId() != null) { - builder.setCustomerIdMSB(user.getCustomerId().getId().getMostSignificantBits()); - builder.setCustomerIdLSB(user.getCustomerId().getId().getLeastSignificantBits()); - } - if (user.getFirstName() != null) { - builder.setFirstName(user.getFirstName()); - } - if (user.getLastName() != null) { - builder.setLastName(user.getLastName()); - } - if (user.getAdditionalInfo() != null) { - builder.setAdditionalInfo(JacksonUtil.toString(user.getAdditionalInfo())); - } - return builder.build(); - } - - @Override - public UserCredentialsUpdateMsg constructUserCredentialsUpdatedMsg(UserCredentials userCredentials) { - return UserCredentialsUpdateMsg.newBuilder() - .setUserIdMSB(userCredentials.getUserId().getId().getMostSignificantBits()) - .setUserIdLSB(userCredentials.getUserId().getId().getLeastSignificantBits()) - .setEnabled(userCredentials.isEnabled()) - .setPassword(userCredentials.getPassword()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorV2.java deleted file mode 100644 index e26e085a3b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/user/UserMsgConstructorV2.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.user; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class UserMsgConstructorV2 extends BaseUserMsgConstructor { - - @Override - public UserUpdateMsg constructUserUpdatedMsg(UpdateMsgType msgType, User user) { - return UserUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(user)) - .setIdMSB(user.getId().getId().getMostSignificantBits()) - .setIdLSB(user.getId().getId().getLeastSignificantBits()).build(); - } - - @Override - public UserCredentialsUpdateMsg constructUserCredentialsUpdatedMsg(UserCredentials userCredentials) { - return UserCredentialsUpdateMsg.newBuilder().setEntity(JacksonUtil.toString(userCredentials)).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/BaseWidgetMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/BaseWidgetMsgConstructor.java deleted file mode 100644 index 5ccf0cf85f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/BaseWidgetMsgConstructor.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.widget; - -import org.thingsboard.server.common.data.id.WidgetTypeId; -import org.thingsboard.server.common.data.id.WidgetsBundleId; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; -import org.thingsboard.server.gen.edge.v1.WidgetsBundleUpdateMsg; - -public abstract class BaseWidgetMsgConstructor implements WidgetMsgConstructor { - - @Override - public WidgetsBundleUpdateMsg constructWidgetsBundleDeleteMsg(WidgetsBundleId widgetsBundleId) { - return WidgetsBundleUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(widgetsBundleId.getId().getMostSignificantBits()) - .setIdLSB(widgetsBundleId.getId().getLeastSignificantBits()) - .build(); - } - - @Override - public WidgetTypeUpdateMsg constructWidgetTypeDeleteMsg(WidgetTypeId widgetTypeId) { - return WidgetTypeUpdateMsg.newBuilder() - .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) - .setIdMSB(widgetTypeId.getId().getMostSignificantBits()) - .setIdLSB(widgetTypeId.getId().getLeastSignificantBits()) - .build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructor.java deleted file mode 100644 index b5f78026bd..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructor.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.widget; - -import org.thingsboard.server.common.data.id.WidgetTypeId; -import org.thingsboard.server.common.data.id.WidgetsBundleId; -import org.thingsboard.server.common.data.widget.WidgetTypeDetails; -import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; -import org.thingsboard.server.gen.edge.v1.WidgetsBundleUpdateMsg; -import org.thingsboard.server.service.edge.rpc.constructor.MsgConstructor; - -import java.util.List; - -public interface WidgetMsgConstructor extends MsgConstructor { - - WidgetsBundleUpdateMsg constructWidgetsBundleUpdateMsg(UpdateMsgType msgType, WidgetsBundle widgetsBundle, List widgets); - - WidgetsBundleUpdateMsg constructWidgetsBundleDeleteMsg(WidgetsBundleId widgetsBundleId); - - WidgetTypeUpdateMsg constructWidgetTypeUpdateMsg(UpdateMsgType msgType, WidgetTypeDetails widgetTypeDetails, EdgeVersion edgeVersion); - - WidgetTypeUpdateMsg constructWidgetTypeDeleteMsg(WidgetTypeId widgetTypeId); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorFactory.java deleted file mode 100644 index 421523c3d8..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.widget; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.BaseMsgConstructorFactory; - -@Component -@TbCoreComponent -public class WidgetMsgConstructorFactory extends BaseMsgConstructorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV1.java deleted file mode 100644 index 0201f6276b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV1.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.widget; - -import com.google.protobuf.ByteString; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.widget.WidgetTypeDetails; -import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.dao.resource.ImageService; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; -import org.thingsboard.server.gen.edge.v1.WidgetsBundleUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.utils.EdgeVersionUtils; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.List; - -@Component -@TbCoreComponent -public class WidgetMsgConstructorV1 extends BaseWidgetMsgConstructor { - - @Autowired - private ImageService imageService; - - @Override - public WidgetsBundleUpdateMsg constructWidgetsBundleUpdateMsg(UpdateMsgType msgType, WidgetsBundle widgetsBundle, List widgets) { - widgetsBundle = JacksonUtil.clone(widgetsBundle); - imageService.inlineImageForEdge(widgetsBundle); - WidgetsBundleUpdateMsg.Builder builder = WidgetsBundleUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(widgetsBundle.getId().getId().getMostSignificantBits()) - .setIdLSB(widgetsBundle.getId().getId().getLeastSignificantBits()) - .setTitle(widgetsBundle.getTitle()) - .setAlias(widgetsBundle.getAlias()); - if (widgetsBundle.getImage() != null) { - builder.setImage(ByteString.copyFrom(widgetsBundle.getImage().getBytes(StandardCharsets.UTF_8))); - } - if (widgetsBundle.getDescription() != null) { - builder.setDescription(widgetsBundle.getDescription()); - } - if (widgetsBundle.getOrder() != null) { - builder.setOrder(widgetsBundle.getOrder()); - } - if (widgetsBundle.getTenantId().equals(TenantId.SYS_TENANT_ID)) { - builder.setIsSystem(true); - } - builder.setWidgets(JacksonUtil.toString(widgets)); - return builder.build(); - } - - @Override - public WidgetTypeUpdateMsg constructWidgetTypeUpdateMsg(UpdateMsgType msgType, WidgetTypeDetails widgetTypeDetails, EdgeVersion edgeVersion) { - widgetTypeDetails = JacksonUtil.clone(widgetTypeDetails); - imageService.inlineImagesForEdge(widgetTypeDetails); - WidgetTypeUpdateMsg.Builder builder = WidgetTypeUpdateMsg.newBuilder() - .setMsgType(msgType) - .setIdMSB(widgetTypeDetails.getId().getId().getMostSignificantBits()) - .setIdLSB(widgetTypeDetails.getId().getId().getLeastSignificantBits()); - if (widgetTypeDetails.getFqn() != null) { - builder.setFqn(widgetTypeDetails.getFqn()); - if (widgetTypeDetails.getFqn().contains(".")) { - String[] aliases = widgetTypeDetails.getFqn().split("\\.", 2); - if (aliases.length == 2) { - builder.setBundleAlias(aliases[0]); - builder.setAlias(aliases[1]); - } - } - } - if (widgetTypeDetails.getName() != null) { - builder.setName(widgetTypeDetails.getName()); - } - if (widgetTypeDetails.getDescriptor() != null) { - builder.setDescriptorJson(JacksonUtil.toString(widgetTypeDetails.getDescriptor())); - } - if (widgetTypeDetails.getTenantId().equals(TenantId.SYS_TENANT_ID)) { - builder.setIsSystem(true); - } - if (widgetTypeDetails.getImage() != null) { - builder.setImage(widgetTypeDetails.getImage()); - } - if (widgetTypeDetails.getDescription() != null) { - if (EdgeVersionUtils.isEdgeVersionOlderThan(edgeVersion, EdgeVersion.V_3_6_0) && - widgetTypeDetails.getDescription().length() > 255) { - builder.setDescription(widgetTypeDetails.getDescription().substring(0, 254)); - } else { - builder.setDescription(widgetTypeDetails.getDescription()); - } - } - builder.setDeprecated(widgetTypeDetails.isDeprecated()); - if (widgetTypeDetails.getTags() != null) { - builder.addAllTags(Arrays.asList(widgetTypeDetails.getTags())); - } - return builder.build(); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV2.java deleted file mode 100644 index 1b1d7e246e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV2.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.constructor.widget; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.widget.WidgetTypeDetails; -import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; -import org.thingsboard.server.gen.edge.v1.WidgetsBundleUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.List; - -@Component -@TbCoreComponent -public class WidgetMsgConstructorV2 extends BaseWidgetMsgConstructor { - - @Override - public WidgetsBundleUpdateMsg constructWidgetsBundleUpdateMsg(UpdateMsgType msgType, WidgetsBundle widgetsBundle, List widgets) { - return WidgetsBundleUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(widgetsBundle)) - .setWidgets(JacksonUtil.toString(widgets)) - .setIdMSB(widgetsBundle.getId().getId().getMostSignificantBits()) - .setIdLSB(widgetsBundle.getId().getId().getLeastSignificantBits()).build(); - } - - @Override - public WidgetTypeUpdateMsg constructWidgetTypeUpdateMsg(UpdateMsgType msgType, WidgetTypeDetails widgetTypeDetails, EdgeVersion edgeVersion) { - return WidgetTypeUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(widgetTypeDetails)) - .setIdMSB(widgetTypeDetails.getId().getId().getMostSignificantBits()) - .setIdLSB(widgetTypeDetails.getId().getId().getLeastSignificantBits()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index e448fbd937..c8647fb40a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -70,7 +70,7 @@ import java.util.concurrent.locks.ReentrantLock; import static org.thingsboard.server.dao.edge.BaseRelatedEdgesService.RELATED_EDGES_CACHE_ITEMS; @Slf4j -public abstract class BaseEdgeProcessor { +public abstract class BaseEdgeProcessor implements EdgeProcessor { protected static final Lock deviceCreationLock = new ReentrantLock(); protected static final Lock assetCreationLock = new ReentrantLock(); @@ -187,6 +187,7 @@ public abstract class BaseEdgeProcessor { }; } + @Override public ListenableFuture processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { EdgeEventType type = EdgeEventType.valueOf(edgeNotificationMsg.getType()); EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessorFactory.java deleted file mode 100644 index a8ca885c7b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessorFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public abstract class BaseEdgeProcessorFactory { - - @Autowired - protected T v1Processor; - - @Autowired - protected U v2Processor; - - public EdgeProcessor getProcessorByEdgeVersion(EdgeVersion edgeVersion) { - return switch (edgeVersion) { - case V_3_3_0, V_3_3_3, V_3_4_0, V_3_6_0, V_3_6_1 -> v1Processor; - default -> v2Processor; - }; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeProcessor.java index dc2ea115ee..b4d4550966 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeProcessor.java @@ -15,5 +15,18 @@ */ package org.thingsboard.server.service.edge.rpc.processor; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.transport.TransportProtos; + public interface EdgeProcessor { + + ListenableFuture processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto msg); + + default DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { + return null; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java index f0a8012660..6cfe5a6819 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java @@ -15,44 +15,35 @@ */ package org.thingsboard.server.service.edge.rpc.processor.alarm; -import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterableByTenantIdEntityId; import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.service.edge.rpc.constructor.alarm.AlarmMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.alarm.AlarmMsgConstructorFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; -import java.util.ArrayList; import java.util.List; import java.util.UUID; -import static org.thingsboard.server.dao.edge.BaseRelatedEdgesService.RELATED_EDGES_CACHE_ITEMS; - @Slf4j -public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements AlarmProcessor { - - @Autowired - private AlarmMsgConstructorFactory alarmMsgConstructorFactory; +@Component +@TbCoreComponent +public class AlarmEdgeProcessor extends BaseAlarmProcessor implements AlarmProcessor { @Autowired private EntityService entityService; @@ -69,9 +60,8 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A } @Override - public DownlinkMsg convertAlarmEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { - AlarmUpdateMsg alarmUpdateMsg = convertAlarmEventToAlarmMsg(edgeEvent.getTenantId(), edgeEvent.getEntityId(), - edgeEvent.getAction(), edgeEvent.getBody(), edgeVersion); + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { + AlarmUpdateMsg alarmUpdateMsg = convertAlarmEventToAlarmMsg(edgeEvent); if (alarmUpdateMsg != null) { return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) @@ -82,37 +72,7 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A } @Override - public ListenableFuture processAlarmCommentMsgFromEdge(TenantId tenantId, EdgeId edgeId, AlarmCommentUpdateMsg alarmCommentUpdateMsg) { - log.trace("[{}] processAlarmCommentMsgFromEdge [{}]", tenantId, alarmCommentUpdateMsg); - try { - edgeSynchronizationManager.getEdgeId().set(edgeId); - return processAlarmCommentMsg(tenantId, alarmCommentUpdateMsg); - } finally { - edgeSynchronizationManager.getEdgeId().remove(); - } - } - - @Override - public DownlinkMsg convertAlarmCommentEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { - UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - var msgConstructor = (AlarmMsgConstructor) alarmMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); - switch (edgeEvent.getAction()) { - case ADDED_COMMENT: - case UPDATED_COMMENT: - case DELETED_COMMENT: - AlarmComment alarmComment = JacksonUtil.convertValue(edgeEvent.getBody(), AlarmComment.class); - if (alarmComment != null) { - return DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addAlarmCommentUpdateMsg(msgConstructor.constructAlarmCommentUpdatedMsg(msgType, alarmComment)) - .build(); - } - default: - return null; - } - } - - public ListenableFuture processAlarmNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { + public ListenableFuture processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()); AlarmId alarmId = new AlarmId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB())); EdgeId originatorEdgeId = safeGetEdgeId(edgeNotificationMsg.getOriginatorEdgeIdMSB(), edgeNotificationMsg.getOriginatorEdgeIdLSB()); @@ -140,51 +100,20 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A }, dbCallbackExecutorService); } - public ListenableFuture processAlarmCommentNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { - EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()); - AlarmId alarmId = new AlarmId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB())); - EdgeId originatorEdgeId = safeGetEdgeId(edgeNotificationMsg.getOriginatorEdgeIdMSB(), edgeNotificationMsg.getOriginatorEdgeIdLSB()); - AlarmComment alarmComment = JacksonUtil.fromString(edgeNotificationMsg.getBody(), AlarmComment.class); - if (alarmComment == null) { - return Futures.immediateFuture(null); - } - Alarm alarmById = edgeCtx.getAlarmService().findAlarmById(tenantId, new AlarmId(alarmComment.getAlarmId().getId())); - List> delFutures = pushEventToAllRelatedEdges(tenantId, alarmById.getOriginator(), - alarmId, actionType, JacksonUtil.valueToTree(alarmComment), originatorEdgeId, EdgeEventType.ALARM_COMMENT); - return Futures.transform(Futures.allAsList(delFutures), voids -> null, dbCallbackExecutorService); - } - - private List> pushEventToAllRelatedEdges(TenantId tenantId, EntityId originatorId, AlarmId alarmId, - EdgeEventActionType actionType, JsonNode body, EdgeId sourceEdgeId, - EdgeEventType edgeEventType) { - List> futures = new ArrayList<>(); - PageDataIterableByTenantIdEntityId edgeIds = - new PageDataIterableByTenantIdEntityId<>(edgeCtx.getEdgeService()::findRelatedEdgeIdsByEntityId, tenantId, originatorId, RELATED_EDGES_CACHE_ITEMS); - for (EdgeId relatedEdgeId : edgeIds) { - if (!relatedEdgeId.equals(sourceEdgeId)) { - futures.add(saveEdgeEvent(tenantId, relatedEdgeId, edgeEventType, actionType, alarmId, body)); - } - } - return futures; - } - - private AlarmUpdateMsg convertAlarmEventToAlarmMsg(TenantId tenantId, UUID entityId, EdgeEventActionType actionType, JsonNode body, EdgeVersion edgeVersion) { - AlarmId alarmId = new AlarmId(entityId); - UpdateMsgType msgType = getUpdateMsgType(actionType); - var msgConstructor = (AlarmMsgConstructor) alarmMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); - switch (actionType) { + private AlarmUpdateMsg convertAlarmEventToAlarmMsg(EdgeEvent edgeEvent) { + AlarmId alarmId = new AlarmId(edgeEvent.getEntityId()); + UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + switch (edgeEvent.getAction()) { case ADDED, UPDATED, ALARM_ACK, ALARM_CLEAR -> { - Alarm alarm = edgeCtx.getAlarmService().findAlarmById(tenantId, alarmId); + Alarm alarm = edgeCtx.getAlarmService().findAlarmById(edgeEvent.getTenantId(), alarmId); if (alarm != null) { - return msgConstructor.constructAlarmUpdatedMsg(msgType, alarm, - entityService.fetchEntityName(tenantId, alarm.getOriginator()).orElse(null)); + return EdgeMsgConstructorUtils.constructAlarmUpdatedMsg(msgType, alarm); } } case ALARM_DELETE, DELETED -> { - Alarm deletedAlarm = JacksonUtil.convertValue(body, Alarm.class); + Alarm deletedAlarm = JacksonUtil.convertValue(edgeEvent.getBody(), Alarm.class); if (deletedAlarm != null) { - return msgConstructor.constructAlarmUpdatedMsg(msgType, deletedAlarm, - entityService.fetchEntityName(tenantId, deletedAlarm.getOriginator()).orElse(null)); + return EdgeMsgConstructorUtils.constructAlarmUpdatedMsg(msgType, deletedAlarm); } } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorFactory.java deleted file mode 100644 index 6d98c01760..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.alarm; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessorFactory; - -@Component -@TbCoreComponent -public class AlarmEdgeProcessorFactory extends BaseEdgeProcessorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV1.java deleted file mode 100644 index 18e65e6bac..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV1.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.alarm; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmSeverity; -import org.thingsboard.server.common.data.alarm.AlarmStatus; -import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class AlarmEdgeProcessorV1 extends AlarmEdgeProcessor { - - @Override - protected Alarm constructAlarmFromUpdateMsg(TenantId tenantId, AlarmId alarmId, EntityId originatorId, AlarmUpdateMsg alarmUpdateMsg) { - Alarm alarm = new Alarm(); - alarm.setId(alarmId); - alarm.setTenantId(tenantId); - alarm.setOriginator(originatorId); - alarm.setType(alarmUpdateMsg.getName()); - alarm.setSeverity(AlarmSeverity.valueOf(alarmUpdateMsg.getSeverity())); - alarm.setStartTs(alarmUpdateMsg.getStartTs()); - AlarmStatus alarmStatus = AlarmStatus.valueOf(alarmUpdateMsg.getStatus()); - alarm.setClearTs(alarmUpdateMsg.getClearTs()); - alarm.setPropagate(alarmUpdateMsg.getPropagate()); - alarm.setCleared(alarmStatus.isCleared()); - alarm.setAcknowledged(alarmStatus.isAck()); - alarm.setAckTs(alarmUpdateMsg.getAckTs()); - alarm.setEndTs(alarmUpdateMsg.getEndTs()); - alarm.setDetails(JacksonUtil.toJsonNode(alarmUpdateMsg.getDetails())); - return alarm; - } - - @Override - protected EntityId getAlarmOriginatorFromMsg(TenantId tenantId, AlarmUpdateMsg alarmUpdateMsg) { - return getAlarmOriginator(tenantId, alarmUpdateMsg.getOriginatorName(), - EntityType.valueOf(alarmUpdateMsg.getOriginatorType())); - } - - private EntityId getAlarmOriginator(TenantId tenantId, String entityName, EntityType entityType) { - return switch (entityType) { - case DEVICE -> edgeCtx.getDeviceService().findDeviceByTenantIdAndName(tenantId, entityName).getId(); - case ASSET -> edgeCtx.getAssetService().findAssetByTenantIdAndName(tenantId, entityName).getId(); - case ENTITY_VIEW -> edgeCtx.getEntityViewService().findEntityViewByTenantIdAndName(tenantId, entityName).getId(); - default -> null; - }; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV2.java deleted file mode 100644 index dd49af252d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV2.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.alarm; - -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Primary -@Component -@TbCoreComponent -public class AlarmEdgeProcessorV2 extends AlarmEdgeProcessor { - - @Override - protected EntityId getAlarmOriginatorFromMsg(TenantId tenantId, AlarmUpdateMsg alarmUpdateMsg) { - Alarm alarm = JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); - return alarm != null ? alarm.getOriginator() : null; - } - - @Override - protected Alarm constructAlarmFromUpdateMsg(TenantId tenantId, AlarmId alarmId, EntityId originatorId, AlarmUpdateMsg alarmUpdateMsg) { - return JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmProcessor.java index d4f820b6c3..8d31c07960 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmProcessor.java @@ -16,23 +16,13 @@ package org.thingsboard.server.service.edge.rpc.processor.alarm; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; public interface AlarmProcessor extends EdgeProcessor { ListenableFuture processAlarmMsgFromEdge(TenantId tenantId, EdgeId edgeId, AlarmUpdateMsg alarmUpdateMsg); - DownlinkMsg convertAlarmEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); - - ListenableFuture processAlarmCommentMsgFromEdge(TenantId tenantId, EdgeId edgeId, AlarmCommentUpdateMsg alarmCommentUpdateMsg); - - DownlinkMsg convertAlarmCommentEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java index 195b569ab4..d806049888 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.edge.rpc.processor.alarm; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; @@ -24,16 +25,24 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.alarm.AlarmCreateOrUpdateActiveRequest; import org.thingsboard.server.common.data.alarm.AlarmUpdateRequest; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterableByTenantIdEntityId; import org.thingsboard.server.dao.alarm.AlarmCommentDao; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; +import static org.thingsboard.server.dao.edge.BaseRelatedEdgesService.RELATED_EDGES_CACHE_ITEMS; + @Slf4j public abstract class BaseAlarmProcessor extends BaseEdgeProcessor { @@ -43,8 +52,7 @@ public abstract class BaseAlarmProcessor extends BaseEdgeProcessor { public ListenableFuture processAlarmMsg(TenantId tenantId, AlarmUpdateMsg alarmUpdateMsg) { log.trace("[{}] processAlarmMsg [{}]", tenantId, alarmUpdateMsg); AlarmId alarmId = new AlarmId(new UUID(alarmUpdateMsg.getIdMSB(), alarmUpdateMsg.getIdLSB())); - EntityId originatorId = getAlarmOriginatorFromMsg(tenantId, alarmUpdateMsg); - Alarm alarm = constructAlarmFromUpdateMsg(tenantId, alarmId, originatorId, alarmUpdateMsg); + Alarm alarm = JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); if (alarm == null) { throw new RuntimeException("[{" + tenantId + "}] alarmUpdateMsg {" + alarmUpdateMsg + "} cannot be converted to alarm"); } @@ -124,9 +132,18 @@ public abstract class BaseAlarmProcessor extends BaseEdgeProcessor { return Futures.immediateFuture(null); } - - protected abstract EntityId getAlarmOriginatorFromMsg(TenantId tenantId, AlarmUpdateMsg alarmUpdateMsg); - - protected abstract Alarm constructAlarmFromUpdateMsg(TenantId tenantId, AlarmId alarmId, EntityId originatorId, AlarmUpdateMsg alarmUpdateMsg); + protected List> pushEventToAllRelatedEdges(TenantId tenantId, EntityId originatorId, AlarmId alarmId, + EdgeEventActionType actionType, JsonNode body, EdgeId sourceEdgeId, + EdgeEventType edgeEventType) { + List> futures = new ArrayList<>(); + PageDataIterableByTenantIdEntityId edgeIds = + new PageDataIterableByTenantIdEntityId<>(edgeCtx.getEdgeService()::findRelatedEdgeIdsByEntityId, tenantId, originatorId, RELATED_EDGES_CACHE_ITEMS); + for (EdgeId relatedEdgeId : edgeIds) { + if (!relatedEdgeId.equals(sourceEdgeId)) { + futures.add(saveEdgeEvent(tenantId, relatedEdgeId, edgeEventType, actionType, alarmId, body)); + } + } + return futures; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentEdgeProcessor.java new file mode 100644 index 0000000000..ef13144ba8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentEdgeProcessor.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.alarm.comment; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmComment; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; +import org.thingsboard.server.service.edge.rpc.processor.alarm.BaseAlarmProcessor; + +import java.util.List; +import java.util.UUID; + +@Slf4j +@Component +@TbCoreComponent +public class AlarmCommentEdgeProcessor extends BaseAlarmProcessor implements AlarmCommentProcessor { + + @Override + public ListenableFuture processAlarmCommentMsgFromEdge(TenantId tenantId, EdgeId edgeId, AlarmCommentUpdateMsg alarmCommentUpdateMsg) { + log.trace("[{}] processAlarmCommentMsgFromEdge [{}]", tenantId, alarmCommentUpdateMsg); + try { + edgeSynchronizationManager.getEdgeId().set(edgeId); + return processAlarmCommentMsg(tenantId, alarmCommentUpdateMsg); + } finally { + edgeSynchronizationManager.getEdgeId().remove(); + } + } + + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { + UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + switch (edgeEvent.getAction()) { + case ADDED_COMMENT: + case UPDATED_COMMENT: + case DELETED_COMMENT: + AlarmComment alarmComment = JacksonUtil.convertValue(edgeEvent.getBody(), AlarmComment.class); + if (alarmComment != null) { + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addAlarmCommentUpdateMsg(EdgeMsgConstructorUtils.constructAlarmCommentUpdatedMsg(msgType, alarmComment)) + .build(); + } + default: + return null; + } + } + + @Override + public ListenableFuture processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { + EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()); + AlarmId alarmId = new AlarmId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB())); + EdgeId originatorEdgeId = safeGetEdgeId(edgeNotificationMsg.getOriginatorEdgeIdMSB(), edgeNotificationMsg.getOriginatorEdgeIdLSB()); + AlarmComment alarmComment = JacksonUtil.fromString(edgeNotificationMsg.getBody(), AlarmComment.class); + if (alarmComment == null) { + return Futures.immediateFuture(null); + } + Alarm alarmById = edgeCtx.getAlarmService().findAlarmById(tenantId, new AlarmId(alarmComment.getAlarmId().getId())); + List> delFutures = pushEventToAllRelatedEdges(tenantId, alarmById.getOriginator(), + alarmId, actionType, JacksonUtil.valueToTree(alarmComment), originatorEdgeId, EdgeEventType.ALARM_COMMENT); + return Futures.transform(Futures.allAsList(delFutures), voids -> null, dbCallbackExecutorService); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/BaseAlarmMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentProcessor.java similarity index 54% rename from application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/BaseAlarmMsgConstructor.java rename to application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentProcessor.java index c9af86e8cc..5be3c0b806 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/alarm/BaseAlarmMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentProcessor.java @@ -13,17 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.edge.rpc.constructor.alarm; +package org.thingsboard.server.service.edge.rpc.processor.alarm.comment; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.alarm.AlarmComment; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; -public abstract class BaseAlarmMsgConstructor implements AlarmMsgConstructor { +public interface AlarmCommentProcessor extends EdgeProcessor { + + ListenableFuture processAlarmCommentMsgFromEdge(TenantId tenantId, EdgeId edgeId, AlarmCommentUpdateMsg alarmCommentUpdateMsg); - @Override - public AlarmCommentUpdateMsg constructAlarmCommentUpdatedMsg(UpdateMsgType msgType, AlarmComment alarmComment) { - return AlarmCommentUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(alarmComment)).build(); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java index 56a4864763..81145a6195 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.data.util.Pair; +import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.asset.Asset; @@ -28,20 +29,23 @@ import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.asset.AssetMsgConstructor; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import java.util.UUID; @Slf4j -public abstract class AssetEdgeProcessor extends BaseAssetProcessor implements AssetProcessor { +@Component +@TbCoreComponent +public class AssetEdgeProcessor extends BaseAssetProcessor implements AssetProcessor { @Override public ListenableFuture processAssetMsgFromEdge(TenantId tenantId, Edge edge, AssetUpdateMsg assetUpdateMsg) { @@ -103,21 +107,20 @@ public abstract class AssetEdgeProcessor extends BaseAssetProcessor implements A } @Override - public DownlinkMsg convertAssetEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { AssetId assetId = new AssetId(edgeEvent.getEntityId()); - var msgConstructor = (AssetMsgConstructor) edgeCtx.getAssetMsgConstructorFactory().getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED, ASSIGNED_TO_EDGE, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER -> { Asset asset = edgeCtx.getAssetService().findAssetById(edgeEvent.getTenantId(), assetId); if (asset != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - AssetUpdateMsg assetUpdateMsg = msgConstructor.constructAssetUpdatedMsg(msgType, asset); + AssetUpdateMsg assetUpdateMsg = EdgeMsgConstructorUtils.constructAssetUpdatedMsg(msgType, asset); DownlinkMsg.Builder builder = DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addAssetUpdateMsg(assetUpdateMsg); if (UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE.equals(msgType)) { AssetProfile assetProfile = edgeCtx.getAssetProfileService().findAssetProfileById(edgeEvent.getTenantId(), asset.getAssetProfileId()); - builder.addAssetProfileUpdateMsg(msgConstructor.constructAssetProfileUpdatedMsg(msgType, assetProfile)); + builder.addAssetProfileUpdateMsg(EdgeMsgConstructorUtils.constructAssetProfileUpdatedMsg(msgType, assetProfile)); } return builder.build(); } @@ -125,11 +128,17 @@ public abstract class AssetEdgeProcessor extends BaseAssetProcessor implements A case DELETED, UNASSIGNED_FROM_EDGE -> { return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addAssetUpdateMsg(msgConstructor.constructAssetDeleteMsg(assetId)) + .addAssetUpdateMsg(EdgeMsgConstructorUtils.constructAssetDeleteMsg(assetId)) .build(); } } return null; } + @Override + protected void setCustomerId(TenantId tenantId, CustomerId customerId, Asset asset, AssetUpdateMsg assetUpdateMsg) { + CustomerId customerUUID = asset.getCustomerId() != null ? asset.getCustomerId() : customerId; + asset.setCustomerId(customerUUID); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorFactory.java deleted file mode 100644 index 150d21d1d0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.asset; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessorFactory; - -@Component -@TbCoreComponent -public class AssetEdgeProcessorFactory extends BaseEdgeProcessorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorV1.java deleted file mode 100644 index 8cd2de1cab..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorV1.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.asset; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.UUID; - -@Component -@TbCoreComponent -public class AssetEdgeProcessorV1 extends AssetEdgeProcessor { - - @Override - protected Asset constructAssetFromUpdateMsg(TenantId tenantId, AssetId assetId, AssetUpdateMsg assetUpdateMsg) { - Asset asset = new Asset(); - asset.setTenantId(tenantId); - asset.setName(assetUpdateMsg.getName()); - asset.setCreatedTime(Uuids.unixTimestamp(assetId.getId())); - asset.setType(assetUpdateMsg.getType()); - asset.setLabel(assetUpdateMsg.hasLabel() ? assetUpdateMsg.getLabel() : null); - asset.setAdditionalInfo(assetUpdateMsg.hasAdditionalInfo() - ? JacksonUtil.toJsonNode(assetUpdateMsg.getAdditionalInfo()) : null); - - UUID assetProfileUUID = safeGetUUID(assetUpdateMsg.getAssetProfileIdMSB(), assetUpdateMsg.getAssetProfileIdLSB()); - asset.setAssetProfileId(assetProfileUUID != null ? new AssetProfileId(assetProfileUUID) : null); - - CustomerId customerId = safeGetCustomerId(assetUpdateMsg.getCustomerIdMSB(), assetUpdateMsg.getCustomerIdLSB()); - asset.setCustomerId(customerId); - return asset; - } - - @Override - protected void setCustomerId(TenantId tenantId, CustomerId customerId, Asset asset, AssetUpdateMsg assetUpdateMsg) { - CustomerId customerUUID = safeGetCustomerId(assetUpdateMsg.getCustomerIdMSB(), assetUpdateMsg.getCustomerIdLSB()); - asset.setCustomerId(customerUUID != null ? customerUUID : customerId); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorV2.java deleted file mode 100644 index 021507f5d7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessorV2.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.asset; - -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Primary -@Component -@TbCoreComponent -public class AssetEdgeProcessorV2 extends AssetEdgeProcessor { - - @Override - protected Asset constructAssetFromUpdateMsg(TenantId tenantId, AssetId assetId, AssetUpdateMsg assetUpdateMsg) { - return JacksonUtil.fromString(assetUpdateMsg.getEntity(), Asset.class, true); - } - - @Override - protected void setCustomerId(TenantId tenantId, CustomerId customerId, Asset asset, AssetUpdateMsg assetUpdateMsg) { - CustomerId customerUUID = asset.getCustomerId() != null ? asset.getCustomerId() : customerId; - asset.setCustomerId(customerUUID); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProcessor.java index f6b015701c..0ae21df039 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProcessor.java @@ -17,17 +17,12 @@ package org.thingsboard.server.service.edge.rpc.processor.asset; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; public interface AssetProcessor extends EdgeProcessor { ListenableFuture processAssetMsgFromEdge(TenantId tenantId, Edge edge, AssetUpdateMsg assetUpdateMsg); - DownlinkMsg convertAssetEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java index 9ebcd74888..7618a2287c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.asset; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.AssetId; @@ -38,7 +39,7 @@ public abstract class BaseAssetProcessor extends BaseEdgeProcessor { boolean assetNameUpdated = false; assetCreationLock.lock(); try { - Asset asset = constructAssetFromUpdateMsg(tenantId, assetId, assetUpdateMsg); + Asset asset = JacksonUtil.fromString(assetUpdateMsg.getEntity(), Asset.class, true); if (asset == null) { throw new RuntimeException("[{" + tenantId + "}] assetUpdateMsg {" + assetUpdateMsg + " } cannot be converted to asset"); } @@ -74,8 +75,6 @@ public abstract class BaseAssetProcessor extends BaseEdgeProcessor { return Pair.of(created, assetNameUpdated); } - protected abstract Asset constructAssetFromUpdateMsg(TenantId tenantId, AssetId assetId, AssetUpdateMsg assetUpdateMsg); - protected abstract void setCustomerId(TenantId tenantId, CustomerId customerId, Asset asset, AssetUpdateMsg assetUpdateMsg); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java index 1998def9fd..df3d74ceaa 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.data.util.Pair; +import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.asset.AssetProfile; @@ -27,21 +28,24 @@ import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.asset.AssetMsgConstructor; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import java.util.UUID; @Slf4j -public abstract class AssetProfileEdgeProcessor extends BaseAssetProfileProcessor implements AssetProfileProcessor { +@Component +@TbCoreComponent +public class AssetProfileEdgeProcessor extends BaseAssetProfileProcessor implements AssetProfileProcessor { @Override public ListenableFuture processAssetProfileMsgFromEdge(TenantId tenantId, Edge edge, AssetProfileUpdateMsg assetProfileUpdateMsg) { @@ -90,15 +94,14 @@ public abstract class AssetProfileEdgeProcessor extends BaseAssetProfileProcesso } @Override - public DownlinkMsg convertAssetProfileEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { AssetProfileId assetProfileId = new AssetProfileId(edgeEvent.getEntityId()); - var msgConstructor = (AssetMsgConstructor) edgeCtx.getAssetMsgConstructorFactory().getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { AssetProfile assetProfile = edgeCtx.getAssetProfileService().findAssetProfileById(edgeEvent.getTenantId(), assetProfileId); if (assetProfile != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - AssetProfileUpdateMsg assetProfileUpdateMsg = msgConstructor.constructAssetProfileUpdatedMsg(msgType, assetProfile); + AssetProfileUpdateMsg assetProfileUpdateMsg = EdgeMsgConstructorUtils.constructAssetProfileUpdatedMsg(msgType, assetProfile); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addAssetProfileUpdateMsg(assetProfileUpdateMsg) @@ -106,7 +109,7 @@ public abstract class AssetProfileEdgeProcessor extends BaseAssetProfileProcesso } } case DELETED -> { - AssetProfileUpdateMsg assetProfileUpdateMsg = msgConstructor.constructAssetProfileDeleteMsg(assetProfileId); + AssetProfileUpdateMsg assetProfileUpdateMsg = EdgeMsgConstructorUtils.constructAssetProfileDeleteMsg(assetProfileId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addAssetProfileUpdateMsg(assetProfileUpdateMsg) @@ -116,4 +119,21 @@ public abstract class AssetProfileEdgeProcessor extends BaseAssetProfileProcesso return null; } + @Override + protected void setDefaultRuleChainId(TenantId tenantId, AssetProfile assetProfile, RuleChainId ruleChainId) { + assetProfile.setDefaultRuleChainId(ruleChainId); + } + + @Override + protected void setDefaultEdgeRuleChainId(AssetProfile assetProfile, RuleChainId ruleChainId, AssetProfileUpdateMsg assetProfileUpdateMsg) { + UUID defaultEdgeRuleChainUUID = ruleChainId != null ? ruleChainId.getId() : null; + assetProfile.setDefaultEdgeRuleChainId(defaultEdgeRuleChainUUID != null ? new RuleChainId(defaultEdgeRuleChainUUID) : null); + } + + @Override + protected void setDefaultDashboardId(TenantId tenantId, DashboardId dashboardId, AssetProfile assetProfile, AssetProfileUpdateMsg assetProfileUpdateMsg) { + UUID defaultDashboardUUID = assetProfile.getDefaultDashboardId() != null ? assetProfile.getDefaultDashboardId().getId() : (dashboardId != null ? dashboardId.getId() : null); + assetProfile.setDefaultDashboardId(defaultDashboardUUID != null ? new DashboardId(defaultDashboardUUID) : null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorFactory.java deleted file mode 100644 index 8d31a9d39c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.asset.profile; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessorFactory; - -@Component -@TbCoreComponent -public class AssetProfileEdgeProcessorFactory extends BaseEdgeProcessorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorV1.java deleted file mode 100644 index 510b5d834e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorV1.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.asset.profile; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.nio.charset.StandardCharsets; -import java.util.UUID; - -@Component -@TbCoreComponent -public class AssetProfileEdgeProcessorV1 extends AssetProfileEdgeProcessor { - - @Override - protected AssetProfile constructAssetProfileFromUpdateMsg(TenantId tenantId, AssetProfileId assetProfileId, AssetProfileUpdateMsg assetProfileUpdateMsg) { - AssetProfile assetProfile = new AssetProfile(); - assetProfile.setTenantId(tenantId); - assetProfile.setName(assetProfileUpdateMsg.getName()); - assetProfile.setCreatedTime(Uuids.unixTimestamp(assetProfileId.getId())); - assetProfile.setDefault(assetProfileUpdateMsg.getDefault()); - assetProfile.setDefaultQueueName(assetProfileUpdateMsg.hasDefaultQueueName() ? assetProfileUpdateMsg.getDefaultQueueName() : null); - assetProfile.setDescription(assetProfileUpdateMsg.hasDescription() ? assetProfileUpdateMsg.getDescription() : null); - assetProfile.setImage(assetProfileUpdateMsg.hasImage() - ? new String(assetProfileUpdateMsg.getImage().toByteArray(), StandardCharsets.UTF_8) : null); - return assetProfile; - } - - @Override - protected void setDefaultRuleChainId(TenantId tenantId, AssetProfile assetProfile, RuleChainId ruleChainId) { - assetProfile.setDefaultRuleChainId(ruleChainId); - } - - @Override - protected void setDefaultEdgeRuleChainId(AssetProfile assetProfile, RuleChainId ruleChainId, AssetProfileUpdateMsg assetProfileUpdateMsg) { - UUID defaultEdgeRuleChainUUID = safeGetUUID(assetProfileUpdateMsg.getDefaultRuleChainIdMSB(), assetProfileUpdateMsg.getDefaultRuleChainIdLSB()); - assetProfile.setDefaultEdgeRuleChainId(defaultEdgeRuleChainUUID != null ? new RuleChainId(defaultEdgeRuleChainUUID) : ruleChainId); - } - - @Override - protected void setDefaultDashboardId(TenantId tenantId, DashboardId dashboardId, AssetProfile assetProfile, AssetProfileUpdateMsg assetProfileUpdateMsg) { - UUID defaultDashboardUUID = safeGetUUID(assetProfileUpdateMsg.getDefaultDashboardIdMSB(), assetProfileUpdateMsg.getDefaultDashboardIdLSB()); - assetProfile.setDefaultDashboardId(defaultDashboardUUID != null ? new DashboardId(defaultDashboardUUID) : dashboardId); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorV2.java deleted file mode 100644 index 9983026213..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessorV2.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.asset.profile; - -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.UUID; - -@Primary -@Component -@TbCoreComponent -public class AssetProfileEdgeProcessorV2 extends AssetProfileEdgeProcessor { - - @Override - protected AssetProfile constructAssetProfileFromUpdateMsg(TenantId tenantId, AssetProfileId assetProfileId, AssetProfileUpdateMsg assetProfileUpdateMsg) { - return JacksonUtil.fromString(assetProfileUpdateMsg.getEntity(), AssetProfile.class, true); - } - - @Override - protected void setDefaultRuleChainId(TenantId tenantId, AssetProfile assetProfile, RuleChainId ruleChainId) { - assetProfile.setDefaultRuleChainId(ruleChainId); - } - - @Override - protected void setDefaultEdgeRuleChainId(AssetProfile assetProfile, RuleChainId ruleChainId, AssetProfileUpdateMsg assetProfileUpdateMsg) { - UUID defaultEdgeRuleChainUUID = ruleChainId != null ? ruleChainId.getId() : null; - assetProfile.setDefaultEdgeRuleChainId(defaultEdgeRuleChainUUID != null ? new RuleChainId(defaultEdgeRuleChainUUID) : null); - } - - @Override - protected void setDefaultDashboardId(TenantId tenantId, DashboardId dashboardId, AssetProfile assetProfile, AssetProfileUpdateMsg assetProfileUpdateMsg) { - UUID defaultDashboardUUID = assetProfile.getDefaultDashboardId() != null ? assetProfile.getDefaultDashboardId().getId() : (dashboardId != null ? dashboardId.getId() : null); - assetProfile.setDefaultDashboardId(defaultDashboardUUID != null ? new DashboardId(defaultDashboardUUID) : null); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileProcessor.java index 83051c519c..d5670924c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileProcessor.java @@ -17,17 +17,12 @@ package org.thingsboard.server.service.edge.rpc.processor.asset.profile; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; public interface AssetProfileProcessor extends EdgeProcessor { ListenableFuture processAssetProfileMsgFromEdge(TenantId tenantId, Edge edge, AssetProfileUpdateMsg assetProfileUpdateMsg); - DownlinkMsg convertAssetProfileEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/BaseAssetProfileProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/BaseAssetProfileProcessor.java index 2b1440a480..af243e3afe 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/BaseAssetProfileProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/BaseAssetProfileProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.asset.profile; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -39,7 +40,7 @@ public abstract class BaseAssetProfileProcessor extends BaseEdgeProcessor { boolean assetProfileNameUpdated = false; assetCreationLock.lock(); try { - AssetProfile assetProfile = constructAssetProfileFromUpdateMsg(tenantId, assetProfileId, assetProfileUpdateMsg); + AssetProfile assetProfile = JacksonUtil.fromString(assetProfileUpdateMsg.getEntity(), AssetProfile.class, true); if (assetProfile == null) { throw new RuntimeException("[{" + tenantId + "}] assetProfileUpdateMsg {" + assetProfileUpdateMsg + "} cannot be converted to asset profile"); } @@ -81,8 +82,6 @@ public abstract class BaseAssetProfileProcessor extends BaseEdgeProcessor { return Pair.of(created, assetProfileNameUpdated); } - protected abstract AssetProfile constructAssetProfileFromUpdateMsg(TenantId tenantId, AssetProfileId assetProfileId, AssetProfileUpdateMsg assetProfileUpdateMsg); - protected abstract void setDefaultRuleChainId(TenantId tenantId, AssetProfile assetProfile, RuleChainId ruleChainId); protected abstract void setDefaultEdgeRuleChainId(AssetProfile assetProfile, RuleChainId ruleChainId, AssetProfileUpdateMsg assetProfileUpdateMsg); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java index ddf6f25728..2ecec3893e 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.edge.rpc.processor.customer; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EdgeUtils; @@ -33,35 +32,30 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.gen.edge.v1.CustomerUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.customer.CustomerMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.customer.CustomerMsgConstructorFactory; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; import java.util.ArrayList; import java.util.List; import java.util.UUID; -@Component @Slf4j +@Component @TbCoreComponent public class CustomerEdgeProcessor extends BaseEdgeProcessor { - @Autowired - private CustomerMsgConstructorFactory customerMsgConstructorFactory; - - public DownlinkMsg convertCustomerEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { CustomerId customerId = new CustomerId(edgeEvent.getEntityId()); - var msgConstructor = (CustomerMsgConstructor) customerMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { Customer customer = edgeCtx.getCustomerService().findCustomerById(edgeEvent.getTenantId(), customerId); if (customer != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - CustomerUpdateMsg customerUpdateMsg = msgConstructor.constructCustomerUpdatedMsg(msgType, customer); + CustomerUpdateMsg customerUpdateMsg = EdgeMsgConstructorUtils.constructCustomerUpdatedMsg(msgType, customer); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addCustomerUpdateMsg(customerUpdateMsg) @@ -69,7 +63,7 @@ public class CustomerEdgeProcessor extends BaseEdgeProcessor { } } case DELETED -> { - CustomerUpdateMsg customerUpdateMsg = msgConstructor.constructCustomerDeleteMsg(customerId); + CustomerUpdateMsg customerUpdateMsg = EdgeMsgConstructorUtils.constructCustomerDeleteMsg(customerId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addCustomerUpdateMsg(customerUpdateMsg) @@ -79,7 +73,8 @@ public class CustomerEdgeProcessor extends BaseEdgeProcessor { return null; } - public ListenableFuture processCustomerNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { + @Override + public ListenableFuture processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()); EdgeEventType type = EdgeEventType.valueOf(edgeNotificationMsg.getType()); UUID uuid = new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB()); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java index 52277b8527..ae593aca1b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.edge.rpc.processor.dashboard; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.ShortCustomerInfo; import org.thingsboard.server.common.data.id.CustomerId; @@ -36,7 +37,7 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { protected boolean saveOrUpdateDashboard(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg, CustomerId customerId) { boolean created = false; - Dashboard dashboard = constructDashboardFromUpdateMsg(tenantId, dashboardId, dashboardUpdateMsg); + Dashboard dashboard = JacksonUtil.fromString(dashboardUpdateMsg.getEntity(), Dashboard.class, true); if (dashboard == null) { throw new RuntimeException("[{" + tenantId + "}] dashboardUpdateMsg {" + dashboardUpdateMsg + "} cannot be converted to dashboard"); } @@ -86,8 +87,6 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { } } - protected abstract Dashboard constructDashboardFromUpdateMsg(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg); - protected abstract Set filterNonExistingCustomers(TenantId tenantId, Set assignedCustomers); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java index cb54977c3c..e4842a16db 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.dashboard; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EdgeUtils; @@ -32,19 +32,17 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.dashboard.DashboardMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.dashboard.DashboardMsgConstructorFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import java.util.Set; import java.util.UUID; @Slf4j -public abstract class DashboardEdgeProcessor extends BaseDashboardProcessor implements DashboardProcessor { - - @Autowired - private DashboardMsgConstructorFactory dashboardMsgConstructorFactory; +@Component +@TbCoreComponent +public class DashboardEdgeProcessor extends BaseDashboardProcessor implements DashboardProcessor { @Override public ListenableFuture processDashboardMsgFromEdge(TenantId tenantId, Edge edge, DashboardUpdateMsg dashboardUpdateMsg) { @@ -101,15 +99,14 @@ public abstract class DashboardEdgeProcessor extends BaseDashboardProcessor impl } @Override - public DownlinkMsg convertDashboardEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { DashboardId dashboardId = new DashboardId(edgeEvent.getEntityId()); - var msgConstructor = (DashboardMsgConstructor) dashboardMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED, ASSIGNED_TO_EDGE, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER -> { Dashboard dashboard = edgeCtx.getDashboardService().findDashboardById(edgeEvent.getTenantId(), dashboardId); if (dashboard != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - DashboardUpdateMsg dashboardUpdateMsg = msgConstructor.constructDashboardUpdatedMsg(msgType, dashboard); + DashboardUpdateMsg dashboardUpdateMsg = EdgeMsgConstructorUtils.constructDashboardUpdatedMsg(msgType, dashboard); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDashboardUpdateMsg(dashboardUpdateMsg) @@ -117,7 +114,7 @@ public abstract class DashboardEdgeProcessor extends BaseDashboardProcessor impl } } case DELETED, UNASSIGNED_FROM_EDGE -> { - DashboardUpdateMsg dashboardUpdateMsg = msgConstructor.constructDashboardDeleteMsg(dashboardId); + DashboardUpdateMsg dashboardUpdateMsg = EdgeMsgConstructorUtils.constructDashboardDeleteMsg(dashboardId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDashboardUpdateMsg(dashboardUpdateMsg) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorFactory.java deleted file mode 100644 index 0d2e9c16fe..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.dashboard; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessorFactory; - -@Component -@TbCoreComponent -public class DashboardEdgeProcessorFactory extends BaseEdgeProcessorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV1.java deleted file mode 100644 index 20ab6019bf..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV1.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.dashboard; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import com.fasterxml.jackson.core.type.TypeReference; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.ShortCustomerInfo; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.Set; - -@Component -@TbCoreComponent -public class DashboardEdgeProcessorV1 extends DashboardEdgeProcessor { - - @Override - protected Dashboard constructDashboardFromUpdateMsg(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg) { - Dashboard dashboard = new Dashboard(); - dashboard.setTenantId(tenantId); - dashboard.setCreatedTime(Uuids.unixTimestamp(dashboardId.getId())); - dashboard.setTitle(dashboardUpdateMsg.getTitle()); - dashboard.setImage(dashboardUpdateMsg.hasImage() ? dashboardUpdateMsg.getImage() : null); - dashboard.setConfiguration(JacksonUtil.toJsonNode(dashboardUpdateMsg.getConfiguration())); - - Set assignedCustomers; - if (dashboardUpdateMsg.hasAssignedCustomers()) { - assignedCustomers = JacksonUtil.fromString(dashboardUpdateMsg.getAssignedCustomers(), new TypeReference<>() {}); - assignedCustomers = filterNonExistingCustomers(tenantId, assignedCustomers); - dashboard.setAssignedCustomers(assignedCustomers); - } - dashboard.setMobileOrder(dashboardUpdateMsg.hasMobileOrder() ? dashboardUpdateMsg.getMobileOrder() : null); - dashboard.setMobileHide(dashboardUpdateMsg.getMobileHide()); - return dashboard; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV2.java deleted file mode 100644 index 10bfe555a6..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV2.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.dashboard; - -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Primary -@Component -@TbCoreComponent -public class DashboardEdgeProcessorV2 extends DashboardEdgeProcessor { - - @Override - protected Dashboard constructDashboardFromUpdateMsg(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg) { - return JacksonUtil.fromString(dashboardUpdateMsg.getEntity(), Dashboard.class, true); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardProcessor.java index 4f24d759bb..88837aff54 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardProcessor.java @@ -17,16 +17,12 @@ package org.thingsboard.server.service.edge.rpc.processor.dashboard; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; public interface DashboardProcessor extends EdgeProcessor { ListenableFuture processDashboardMsgFromEdge(TenantId tenantId, Edge edge, DashboardUpdateMsg dashboardUpdateMsg); - DownlinkMsg convertDashboardEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java index 4b2757e510..2bea268da1 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.device; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.CustomerId; @@ -40,7 +41,7 @@ public abstract class BaseDeviceProcessor extends BaseEdgeProcessor { boolean deviceNameUpdated = false; deviceCreationLock.lock(); try { - Device device = constructDeviceFromUpdateMsg(tenantId, deviceId, deviceUpdateMsg); + Device device =JacksonUtil.fromString(deviceUpdateMsg.getEntity(), Device.class, true); if (device == null) { throw new RuntimeException("[{" + tenantId + "}] deviceUpdateMsg {" + deviceUpdateMsg + "} cannot be converted to device"); } @@ -78,7 +79,7 @@ public abstract class BaseDeviceProcessor extends BaseEdgeProcessor { } protected void updateDeviceCredentials(TenantId tenantId, DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg) { - DeviceCredentials deviceCredentials = constructDeviceCredentialsFromUpdateMsg(tenantId, deviceCredentialsUpdateMsg); + DeviceCredentials deviceCredentials = JacksonUtil.fromString(deviceCredentialsUpdateMsg.getEntity(), DeviceCredentials.class, true); if (deviceCredentials == null) { throw new RuntimeException("[{" + tenantId + "}] deviceCredentialsUpdateMsg {" + deviceCredentialsUpdateMsg + "} cannot be converted to device credentials"); } @@ -107,10 +108,6 @@ public abstract class BaseDeviceProcessor extends BaseEdgeProcessor { } } - protected abstract Device constructDeviceFromUpdateMsg(TenantId tenantId, DeviceId deviceId, DeviceUpdateMsg deviceUpdateMsg); - protected abstract void setCustomerId(TenantId tenantId, CustomerId customerId, Device device, DeviceUpdateMsg deviceUpdateMsg); - protected abstract DeviceCredentials constructDeviceCredentialsFromUpdateMsg(TenantId tenantId, DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg); - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java index e453cbc26f..daf687fe6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.data.util.Pair; +import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; @@ -47,16 +49,18 @@ import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.service.edge.rpc.constructor.device.DeviceMsgConstructor; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import java.util.UUID; @Slf4j -public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements DeviceProcessor { +@Component +@TbCoreComponent +public class DeviceEdgeProcessor extends BaseDeviceProcessor implements DeviceProcessor { @Override public ListenableFuture processDeviceMsgFromEdge(TenantId tenantId, Edge edge, DeviceUpdateMsg deviceUpdateMsg) { @@ -213,9 +217,8 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements } @Override - public DownlinkMsg convertDeviceEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { DeviceId deviceId = new DeviceId(edgeEvent.getEntityId()); - var msgConstructor = (DeviceMsgConstructor) edgeCtx.getDeviceMsgConstructorFactory().getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED: case UPDATED: @@ -225,18 +228,18 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements Device device = edgeCtx.getDeviceService().findDeviceById(edgeEvent.getTenantId(), deviceId); if (device != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - DeviceUpdateMsg deviceUpdateMsg = msgConstructor.constructDeviceUpdatedMsg(msgType, device); + DeviceUpdateMsg deviceUpdateMsg = EdgeMsgConstructorUtils.constructDeviceUpdatedMsg(msgType, device); DownlinkMsg.Builder builder = DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDeviceUpdateMsg(deviceUpdateMsg); DeviceCredentials deviceCredentials = edgeCtx.getDeviceCredentialsService().findDeviceCredentialsByDeviceId(edgeEvent.getTenantId(), deviceId); if (deviceCredentials != null) { - DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg = msgConstructor.constructDeviceCredentialsUpdatedMsg(deviceCredentials); + DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg = EdgeMsgConstructorUtils.constructDeviceCredentialsUpdatedMsg(deviceCredentials); builder.addDeviceCredentialsUpdateMsg(deviceCredentialsUpdateMsg).build(); } if (UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE.equals(msgType)) { DeviceProfile deviceProfile = edgeCtx.getDeviceProfileService().findDeviceProfileById(edgeEvent.getTenantId(), device.getDeviceProfileId()); - builder.addDeviceProfileUpdateMsg(msgConstructor.constructDeviceProfileUpdatedMsg(msgType, deviceProfile)); + builder.addDeviceProfileUpdateMsg(EdgeMsgConstructorUtils.constructDeviceProfileUpdatedMsg(msgType, deviceProfile)); } return builder.build(); } @@ -245,24 +248,30 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements case UNASSIGNED_FROM_EDGE: return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addDeviceUpdateMsg(msgConstructor.constructDeviceDeleteMsg(deviceId)) + .addDeviceUpdateMsg(EdgeMsgConstructorUtils.constructDeviceDeleteMsg(deviceId)) .build(); case CREDENTIALS_UPDATED: DeviceCredentials deviceCredentials = edgeCtx.getDeviceCredentialsService().findDeviceCredentialsByDeviceId(edgeEvent.getTenantId(), deviceId); if (deviceCredentials != null) { return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addDeviceCredentialsUpdateMsg(msgConstructor.constructDeviceCredentialsUpdatedMsg(deviceCredentials)) + .addDeviceCredentialsUpdateMsg(EdgeMsgConstructorUtils.constructDeviceCredentialsUpdatedMsg(deviceCredentials)) .build(); } break; case RPC_CALL: return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addDeviceRpcCallMsg(msgConstructor.constructDeviceRpcCallMsg(edgeEvent.getEntityId(), edgeEvent.getBody())) + .addDeviceRpcCallMsg(EdgeMsgConstructorUtils.constructDeviceRpcCallMsg(edgeEvent.getEntityId(), edgeEvent.getBody())) .build(); } return null; } + @Override + protected void setCustomerId(TenantId tenantId, CustomerId customerId, Device device, DeviceUpdateMsg deviceUpdateMsg) { + CustomerId customerUUID = device.getCustomerId() != null ? device.getCustomerId() : customerId; + device.setCustomerId(customerUUID); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorFactory.java deleted file mode 100644 index 85327bfc1f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.device; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessorFactory; - -@Component -@TbCoreComponent -public class DeviceEdgeProcessorFactory extends BaseEdgeProcessorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorV1.java deleted file mode 100644 index 6756c6c2f8..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorV1.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.device; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.OtaPackageId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.common.data.security.DeviceCredentialsType; -import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.UUID; - -@Component -@TbCoreComponent -public class DeviceEdgeProcessorV1 extends DeviceEdgeProcessor { - - @Override - protected Device constructDeviceFromUpdateMsg(TenantId tenantId, DeviceId deviceId, DeviceUpdateMsg deviceUpdateMsg) { - Device device = new Device(); - device.setTenantId(tenantId); - device.setCreatedTime(Uuids.unixTimestamp(deviceId.getId())); - device.setName(deviceUpdateMsg.getName()); - device.setType(deviceUpdateMsg.getType()); - device.setLabel(deviceUpdateMsg.hasLabel() ? deviceUpdateMsg.getLabel() : null); - device.setAdditionalInfo(deviceUpdateMsg.hasAdditionalInfo() - ? JacksonUtil.toJsonNode(deviceUpdateMsg.getAdditionalInfo()) : null); - - UUID deviceProfileUUID = safeGetUUID(deviceUpdateMsg.getDeviceProfileIdMSB(), deviceUpdateMsg.getDeviceProfileIdLSB()); - device.setDeviceProfileId(deviceProfileUUID != null ? new DeviceProfileId(deviceProfileUUID) : null); - - device.setDeviceDataBytes(deviceUpdateMsg.getDeviceDataBytes().toByteArray()); - - UUID firmwareUUID = safeGetUUID(deviceUpdateMsg.getFirmwareIdMSB(), deviceUpdateMsg.getFirmwareIdLSB()); - device.setFirmwareId(firmwareUUID != null ? new OtaPackageId(firmwareUUID) : null); - UUID softwareUUID = safeGetUUID(deviceUpdateMsg.getSoftwareIdMSB(), deviceUpdateMsg.getSoftwareIdLSB()); - device.setSoftwareId(softwareUUID != null ? new OtaPackageId(softwareUUID) : null); - - return device; - } - - @Override - protected DeviceCredentials constructDeviceCredentialsFromUpdateMsg(TenantId tenantId, DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg) { - DeviceCredentials deviceCredentials = new DeviceCredentials(); - deviceCredentials.setDeviceId(new DeviceId(new UUID(deviceCredentialsUpdateMsg.getDeviceIdMSB(), deviceCredentialsUpdateMsg.getDeviceIdLSB()))); - deviceCredentials.setCredentialsType(DeviceCredentialsType.valueOf(deviceCredentialsUpdateMsg.getCredentialsType())); - deviceCredentials.setCredentialsId(deviceCredentialsUpdateMsg.getCredentialsId()); - deviceCredentials.setCredentialsValue(deviceCredentialsUpdateMsg.hasCredentialsValue() - ? deviceCredentialsUpdateMsg.getCredentialsValue() : null); - return deviceCredentials; - } - - @Override - protected void setCustomerId(TenantId tenantId, CustomerId customerId, Device device, DeviceUpdateMsg deviceUpdateMsg) { - CustomerId customerUUID = safeGetCustomerId(deviceUpdateMsg.getCustomerIdMSB(), deviceUpdateMsg.getCustomerIdLSB()); - device.setCustomerId(customerUUID != null ? customerUUID : customerId); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorV2.java deleted file mode 100644 index 0ea4f62862..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessorV2.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.device; - -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Primary -@Component -@TbCoreComponent -public class DeviceEdgeProcessorV2 extends DeviceEdgeProcessor { - - @Override - protected Device constructDeviceFromUpdateMsg(TenantId tenantId, DeviceId deviceId, DeviceUpdateMsg deviceUpdateMsg) { - return JacksonUtil.fromString(deviceUpdateMsg.getEntity(), Device.class, true); - } - - @Override - protected DeviceCredentials constructDeviceCredentialsFromUpdateMsg(TenantId tenantId, DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg) { - return JacksonUtil.fromString(deviceCredentialsUpdateMsg.getEntity(), DeviceCredentials.class, true); - } - - @Override - protected void setCustomerId(TenantId tenantId, CustomerId customerId, Device device, DeviceUpdateMsg deviceUpdateMsg) { - CustomerId customerUUID = device.getCustomerId() != null ? device.getCustomerId() : customerId; - device.setCustomerId(customerUUID); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProcessor.java index 087b5f36a7..a7fdbae588 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProcessor.java @@ -17,14 +17,11 @@ package org.thingsboard.server.service.edge.rpc.processor.device; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; public interface DeviceProcessor extends EdgeProcessor { @@ -33,8 +30,6 @@ public interface DeviceProcessor extends EdgeProcessor { ListenableFuture processDeviceCredentialsMsgFromEdge(TenantId tenantId, EdgeId edgeId, DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg); - DownlinkMsg convertDeviceEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); - ListenableFuture processDeviceRpcCallFromEdge(TenantId tenantId, Edge edge, DeviceRpcCallMsg deviceRpcCallMsg); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/BaseDeviceProfileProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/BaseDeviceProfileProcessor.java index d06618772d..47935da4fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/BaseDeviceProfileProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/BaseDeviceProfileProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.device.profile; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DashboardId; @@ -39,7 +40,7 @@ public abstract class BaseDeviceProfileProcessor extends BaseEdgeProcessor { boolean deviceProfileNameUpdated = false; deviceCreationLock.lock(); try { - DeviceProfile deviceProfile = constructDeviceProfileFromUpdateMsg(tenantId, deviceProfileId, deviceProfileUpdateMsg); + DeviceProfile deviceProfile = JacksonUtil.fromString(deviceProfileUpdateMsg.getEntity(), DeviceProfile.class, true); if (deviceProfile == null) { throw new RuntimeException("[{" + tenantId + "}] deviceProfileUpdateMsg {" + deviceProfileUpdateMsg + "} cannot be converted to device profile"); } @@ -81,8 +82,6 @@ public abstract class BaseDeviceProfileProcessor extends BaseEdgeProcessor { return Pair.of(created, deviceProfileNameUpdated); } - protected abstract DeviceProfile constructDeviceProfileFromUpdateMsg(TenantId tenantId, DeviceProfileId deviceProfileId, DeviceProfileUpdateMsg deviceProfileUpdateMsg); - protected abstract void setDefaultRuleChainId(TenantId tenantId, DeviceProfile deviceProfile, RuleChainId ruleChainId); protected abstract void setDefaultEdgeRuleChainId(DeviceProfile deviceProfile, RuleChainId ruleChainId, DeviceProfileUpdateMsg deviceProfileUpdateMsg); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java index a0a699d340..31a007219d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.data.util.Pair; +import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EdgeUtils; @@ -26,21 +27,25 @@ import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.device.DeviceMsgConstructor; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import java.util.UUID; @Slf4j -public abstract class DeviceProfileEdgeProcessor extends BaseDeviceProfileProcessor implements DeviceProfileProcessor { +@Component +@TbCoreComponent +public class DeviceProfileEdgeProcessor extends BaseDeviceProfileProcessor implements DeviceProfileProcessor { @Override public ListenableFuture processDeviceProfileMsgFromEdge(TenantId tenantId, Edge edge, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { @@ -49,16 +54,13 @@ public abstract class DeviceProfileEdgeProcessor extends BaseDeviceProfileProces try { edgeSynchronizationManager.getEdgeId().set(edge.getId()); - switch (deviceProfileUpdateMsg.getMsgType()) { - case ENTITY_CREATED_RPC_MESSAGE: - case ENTITY_UPDATED_RPC_MESSAGE: + return switch (deviceProfileUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE, ENTITY_UPDATED_RPC_MESSAGE -> { saveOrUpdateDeviceProfile(tenantId, deviceProfileId, deviceProfileUpdateMsg, edge); - return Futures.immediateFuture(null); - case ENTITY_DELETED_RPC_MESSAGE: - case UNRECOGNIZED: - default: - return handleUnsupportedMsgType(deviceProfileUpdateMsg.getMsgType()); - } + yield Futures.immediateFuture(null); + } + default -> handleUnsupportedMsgType(deviceProfileUpdateMsg.getMsgType()); + }; } catch (DataValidationException e) { log.warn("[{}] Failed to process DeviceProfileUpdateMsg from Edge [{}]", tenantId, deviceProfileUpdateMsg, e); return Futures.immediateFailedFuture(e); @@ -92,15 +94,14 @@ public abstract class DeviceProfileEdgeProcessor extends BaseDeviceProfileProces } @Override - public DownlinkMsg convertDeviceProfileEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { DeviceProfileId deviceProfileId = new DeviceProfileId(edgeEvent.getEntityId()); - var msgConstructor = (DeviceMsgConstructor) edgeCtx.getDeviceMsgConstructorFactory().getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { DeviceProfile deviceProfile = edgeCtx.getDeviceProfileService().findDeviceProfileById(edgeEvent.getTenantId(), deviceProfileId); if (deviceProfile != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - DeviceProfileUpdateMsg deviceProfileUpdateMsg = msgConstructor.constructDeviceProfileUpdatedMsg(msgType, deviceProfile); + DeviceProfileUpdateMsg deviceProfileUpdateMsg = EdgeMsgConstructorUtils.constructDeviceProfileUpdatedMsg(msgType, deviceProfile); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDeviceProfileUpdateMsg(deviceProfileUpdateMsg) @@ -108,7 +109,7 @@ public abstract class DeviceProfileEdgeProcessor extends BaseDeviceProfileProces } } case DELETED -> { - DeviceProfileUpdateMsg deviceProfileUpdateMsg = msgConstructor.constructDeviceProfileDeleteMsg(deviceProfileId); + DeviceProfileUpdateMsg deviceProfileUpdateMsg = EdgeMsgConstructorUtils.constructDeviceProfileDeleteMsg(deviceProfileId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDeviceProfileUpdateMsg(deviceProfileUpdateMsg) @@ -118,4 +119,21 @@ public abstract class DeviceProfileEdgeProcessor extends BaseDeviceProfileProces return null; } + @Override + protected void setDefaultRuleChainId(TenantId tenantId, DeviceProfile deviceProfile, RuleChainId ruleChainId) { + deviceProfile.setDefaultRuleChainId(ruleChainId); + } + + @Override + protected void setDefaultEdgeRuleChainId(DeviceProfile deviceProfile, RuleChainId ruleChainId, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { + UUID defaultEdgeRuleChainUUID = ruleChainId != null ? ruleChainId.getId() : null; + deviceProfile.setDefaultEdgeRuleChainId(defaultEdgeRuleChainUUID != null ? new RuleChainId(defaultEdgeRuleChainUUID) : null); + } + + @Override + protected void setDefaultDashboardId(TenantId tenantId, DashboardId dashboardId, DeviceProfile deviceProfile, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { + UUID defaultDashboardUUID = deviceProfile.getDefaultDashboardId() != null ? deviceProfile.getDefaultDashboardId().getId() : (dashboardId != null ? dashboardId.getId() : null); + deviceProfile.setDefaultDashboardId(defaultDashboardUUID != null ? new DashboardId(defaultDashboardUUID) : null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorFactory.java deleted file mode 100644 index a69fba1fb5..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.device.profile; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessorFactory; - -@Component -@TbCoreComponent -public class DeviceProfileEdgeProcessorFactory extends BaseEdgeProcessorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorV1.java deleted file mode 100644 index a86afcfb73..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorV1.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.device.profile; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceProfileProvisionType; -import org.thingsboard.server.common.data.DeviceProfileType; -import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.OtaPackageId; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.nio.charset.StandardCharsets; -import java.util.UUID; - -@Component -@TbCoreComponent -public class DeviceProfileEdgeProcessorV1 extends DeviceProfileEdgeProcessor { - - @Override - protected DeviceProfile constructDeviceProfileFromUpdateMsg(TenantId tenantId, DeviceProfileId deviceProfileId, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { - DeviceProfile deviceProfile = new DeviceProfile(); - deviceProfile.setTenantId(tenantId); - deviceProfile.setCreatedTime(Uuids.unixTimestamp(deviceProfileId.getId())); - deviceProfile.setName(deviceProfileUpdateMsg.getName()); - deviceProfile.setDefault(deviceProfileUpdateMsg.getDefault()); - deviceProfile.setDescription(deviceProfileUpdateMsg.hasDescription() ? deviceProfileUpdateMsg.getDescription() : null); - deviceProfile.setType(DeviceProfileType.valueOf(deviceProfileUpdateMsg.getType())); - deviceProfile.setTransportType(deviceProfileUpdateMsg.hasTransportType() - ? DeviceTransportType.valueOf(deviceProfileUpdateMsg.getTransportType()) : DeviceTransportType.DEFAULT); - deviceProfile.setImage(deviceProfileUpdateMsg.hasImage() - ? new String(deviceProfileUpdateMsg.getImage().toByteArray(), StandardCharsets.UTF_8) : null); - deviceProfile.setProvisionType(deviceProfileUpdateMsg.hasProvisionType() - ? DeviceProfileProvisionType.valueOf(deviceProfileUpdateMsg.getProvisionType()) : DeviceProfileProvisionType.DISABLED); - deviceProfile.setProvisionDeviceKey(deviceProfileUpdateMsg.hasProvisionDeviceKey() - ? deviceProfileUpdateMsg.getProvisionDeviceKey() : null); - deviceProfile.setDefaultQueueName(deviceProfileUpdateMsg.getDefaultQueueName()); - - deviceProfile.setProfileDataBytes(deviceProfileUpdateMsg.getProfileDataBytes().toByteArray()); - - String defaultQueueName = StringUtils.isNotBlank(deviceProfileUpdateMsg.getDefaultQueueName()) - ? deviceProfileUpdateMsg.getDefaultQueueName() : null; - deviceProfile.setDefaultQueueName(defaultQueueName); - - UUID firmwareUUID = safeGetUUID(deviceProfileUpdateMsg.getFirmwareIdMSB(), deviceProfileUpdateMsg.getFirmwareIdLSB()); - deviceProfile.setFirmwareId(firmwareUUID != null ? new OtaPackageId(firmwareUUID) : null); - - UUID softwareUUID = safeGetUUID(deviceProfileUpdateMsg.getSoftwareIdMSB(), deviceProfileUpdateMsg.getSoftwareIdLSB()); - deviceProfile.setSoftwareId(softwareUUID != null ? new OtaPackageId(softwareUUID) : null); - return deviceProfile; - } - - @Override - protected void setDefaultRuleChainId(TenantId tenantId, DeviceProfile deviceProfile, RuleChainId ruleChainId) { - deviceProfile.setDefaultRuleChainId(ruleChainId); - } - - @Override - protected void setDefaultEdgeRuleChainId(DeviceProfile deviceProfile, RuleChainId ruleChainId, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { - UUID defaultEdgeRuleChainUUID = safeGetUUID(deviceProfileUpdateMsg.getDefaultRuleChainIdMSB(), deviceProfileUpdateMsg.getDefaultRuleChainIdLSB()); - deviceProfile.setDefaultEdgeRuleChainId(defaultEdgeRuleChainUUID != null ? new RuleChainId(defaultEdgeRuleChainUUID) : ruleChainId); - } - - @Override - protected void setDefaultDashboardId(TenantId tenantId, DashboardId dashboardId, DeviceProfile deviceProfile, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { - UUID defaultDashboardUUID = safeGetUUID(deviceProfileUpdateMsg.getDefaultDashboardIdMSB(), deviceProfileUpdateMsg.getDefaultDashboardIdLSB()); - deviceProfile.setDefaultDashboardId(defaultDashboardUUID != null ? new DashboardId(defaultDashboardUUID) : dashboardId); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorV2.java deleted file mode 100644 index 15c07fdaa7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessorV2.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.device.profile; - -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.UUID; - -@Primary -@Component -@TbCoreComponent -public class DeviceProfileEdgeProcessorV2 extends DeviceProfileEdgeProcessor { - - @Override - protected DeviceProfile constructDeviceProfileFromUpdateMsg(TenantId tenantId, DeviceProfileId deviceProfileId, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { - return JacksonUtil.fromString(deviceProfileUpdateMsg.getEntity(), DeviceProfile.class, true); - } - - @Override - protected void setDefaultRuleChainId(TenantId tenantId, DeviceProfile deviceProfile, RuleChainId ruleChainId) { - deviceProfile.setDefaultRuleChainId(ruleChainId); - } - - @Override - protected void setDefaultEdgeRuleChainId(DeviceProfile deviceProfile, RuleChainId ruleChainId, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { - UUID defaultEdgeRuleChainUUID = ruleChainId != null ? ruleChainId.getId() : null; - deviceProfile.setDefaultEdgeRuleChainId(defaultEdgeRuleChainUUID != null ? new RuleChainId(defaultEdgeRuleChainUUID) : null); - } - - @Override - protected void setDefaultDashboardId(TenantId tenantId, DashboardId dashboardId, DeviceProfile deviceProfile, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { - UUID defaultDashboardUUID = deviceProfile.getDefaultDashboardId() != null ? deviceProfile.getDefaultDashboardId().getId() : (dashboardId != null ? dashboardId.getId() : null); - deviceProfile.setDefaultDashboardId(defaultDashboardUUID != null ? new DashboardId(defaultDashboardUUID) : null); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileProcessor.java index 817d35879d..77bf9500c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileProcessor.java @@ -17,17 +17,12 @@ package org.thingsboard.server.service.edge.rpc.processor.device.profile; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; public interface DeviceProfileProcessor extends EdgeProcessor { ListenableFuture processDeviceProfileMsgFromEdge(TenantId tenantId, Edge edge, DeviceProfileUpdateMsg deviceProfileUpdateMsg); - DownlinkMsg convertDeviceProfileEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeProcessor.java rename to application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java index 024147507a..7c1f907cb0 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java @@ -32,9 +32,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; import java.util.ArrayList; @@ -44,26 +44,10 @@ import java.util.UUID; @Slf4j @Component @TbCoreComponent -public class EdgeProcessor extends BaseEdgeProcessor { +public class EdgeEntityProcessor extends BaseEdgeProcessor { - public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { - EdgeId edgeId = new EdgeId(edgeEvent.getEntityId()); - switch (edgeEvent.getAction()) { - case ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER -> { - Edge edge = edgeCtx.getEdgeService().findEdgeById(edgeEvent.getTenantId(), edgeId); - if (edge != null) { - EdgeConfiguration edgeConfigMsg = edgeCtx.getEdgeMsgConstructor().constructEdgeConfiguration(edge); - return DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .setEdgeConfiguration(edgeConfigMsg) - .build(); - } - } - } - return null; - } - - public ListenableFuture processEdgeNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { + @Override + public ListenableFuture processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { try { EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()); EdgeId edgeId = new EdgeId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB())); @@ -112,4 +96,21 @@ public class EdgeProcessor extends BaseEdgeProcessor { } } + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { + EdgeId edgeId = new EdgeId(edgeEvent.getEntityId()); + switch (edgeEvent.getAction()) { + case ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER -> { + Edge edge = edgeCtx.getEdgeService().findEdgeById(edgeEvent.getTenantId(), edgeId); + if (edge != null) { + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .setEdgeConfiguration(EdgeMsgConstructorUtils.constructEdgeConfiguration(edge)) + .build(); + } + } + } + return null; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java index 757f606f69..2ded93c253 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.entityview; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.CustomerId; @@ -36,7 +37,7 @@ public abstract class BaseEntityViewProcessor extends BaseEdgeProcessor { protected Pair saveOrUpdateEntityView(TenantId tenantId, EntityViewId entityViewId, EntityViewUpdateMsg entityViewUpdateMsg) { boolean created = false; boolean entityViewNameUpdated = false; - EntityView entityView = constructEntityViewFromUpdateMsg(tenantId, entityViewId, entityViewUpdateMsg); + EntityView entityView = JacksonUtil.fromString(entityViewUpdateMsg.getEntity(), EntityView.class, true); if (entityView == null) { throw new RuntimeException("[{" + tenantId + "}] entityViewUpdateMsg {" + entityViewUpdateMsg + "} cannot be converted to entity view"); } @@ -66,8 +67,6 @@ public abstract class BaseEntityViewProcessor extends BaseEdgeProcessor { return Pair.of(created, entityViewNameUpdated); } - protected abstract EntityView constructEntityViewFromUpdateMsg(TenantId tenantId, EntityViewId entityViewId, EntityViewUpdateMsg entityViewUpdateMsg); - protected abstract void setCustomerId(TenantId tenantId, CustomerId customerId, EntityView entityView, EntityViewUpdateMsg entityViewUpdateMsg); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java index aa105652d5..a46ac3acdd 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java @@ -18,8 +18,8 @@ package org.thingsboard.server.service.edge.rpc.processor.entityview; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; +import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityView; @@ -27,25 +27,24 @@ import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.entityview.EntityViewMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.entityview.EntityViewMsgConstructorFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import java.util.UUID; @Slf4j -public abstract class EntityViewEdgeProcessor extends BaseEntityViewProcessor implements EntityViewProcessor { - - @Autowired - private EntityViewMsgConstructorFactory entityViewMsgConstructorFactory; +@Component +@TbCoreComponent +public class EntityViewEdgeProcessor extends BaseEntityViewProcessor implements EntityViewProcessor { @Override public ListenableFuture processEntityViewMsgFromEdge(TenantId tenantId, Edge edge, EntityViewUpdateMsg entityViewUpdateMsg) { @@ -106,15 +105,15 @@ public abstract class EntityViewEdgeProcessor extends BaseEntityViewProcessor im } } - public DownlinkMsg convertEntityViewEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { EntityViewId entityViewId = new EntityViewId(edgeEvent.getEntityId()); - var msgConstructor = (EntityViewMsgConstructor) entityViewMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED, ASSIGNED_TO_EDGE, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER -> { EntityView entityView = edgeCtx.getEntityViewService().findEntityViewById(edgeEvent.getTenantId(), entityViewId); if (entityView != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - EntityViewUpdateMsg entityViewUpdateMsg = msgConstructor.constructEntityViewUpdatedMsg(msgType, entityView); + EntityViewUpdateMsg entityViewUpdateMsg = EdgeMsgConstructorUtils.constructEntityViewUpdatedMsg(msgType, entityView); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addEntityViewUpdateMsg(entityViewUpdateMsg) @@ -122,7 +121,7 @@ public abstract class EntityViewEdgeProcessor extends BaseEntityViewProcessor im } } case DELETED, UNASSIGNED_FROM_EDGE -> { - EntityViewUpdateMsg entityViewUpdateMsg = msgConstructor.constructEntityViewDeleteMsg(entityViewId); + EntityViewUpdateMsg entityViewUpdateMsg = EdgeMsgConstructorUtils.constructEntityViewDeleteMsg(entityViewId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addEntityViewUpdateMsg(entityViewUpdateMsg) @@ -132,4 +131,10 @@ public abstract class EntityViewEdgeProcessor extends BaseEntityViewProcessor im return null; } + @Override + protected void setCustomerId(TenantId tenantId, CustomerId customerId, EntityView entityView, EntityViewUpdateMsg entityViewUpdateMsg) { + CustomerId customerUUID = entityView.getCustomerId() != null ? entityView.getCustomerId() : customerId; + entityView.setCustomerId(customerUUID); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessor.java index 061cd89bf0..b0eaf1d60e 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessor.java @@ -17,10 +17,7 @@ package org.thingsboard.server.service.edge.rpc.processor.entityview; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; @@ -28,6 +25,4 @@ public interface EntityViewProcessor extends EdgeProcessor { ListenableFuture processEntityViewMsgFromEdge(TenantId tenantId, Edge edge, EntityViewUpdateMsg entityViewUpdateMsg); - DownlinkMsg convertEntityViewEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorFactory.java deleted file mode 100644 index d6a3f0c250..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.entityview; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessorFactory; - -@Component -@TbCoreComponent -public class EntityViewProcessorFactory extends BaseEdgeProcessorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorV1.java deleted file mode 100644 index acb8877c03..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorV1.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.entityview; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.EdgeEntityType; -import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.UUID; - -@Component -@TbCoreComponent -public class EntityViewProcessorV1 extends EntityViewEdgeProcessor { - - @Override - protected EntityView constructEntityViewFromUpdateMsg(TenantId tenantId, EntityViewId entityViewId, EntityViewUpdateMsg entityViewUpdateMsg) { - EntityView entityView = new EntityView(); - entityView.setTenantId(tenantId); - entityView.setCreatedTime(Uuids.unixTimestamp(entityViewId.getId())); - entityView.setName(entityViewUpdateMsg.getName()); - entityView.setType(entityViewUpdateMsg.getType()); - - entityView.setAdditionalInfo(entityViewUpdateMsg.hasAdditionalInfo() ? - JacksonUtil.toJsonNode(entityViewUpdateMsg.getAdditionalInfo()) : null); - - CustomerId customerId = safeGetCustomerId(entityViewUpdateMsg.getCustomerIdMSB(), entityViewUpdateMsg.getCustomerIdLSB()); - entityView.setCustomerId(customerId); - - UUID entityIdUUID = safeGetUUID(entityViewUpdateMsg.getEntityIdMSB(), entityViewUpdateMsg.getEntityIdLSB()); - if (EdgeEntityType.DEVICE.equals(entityViewUpdateMsg.getEntityType())) { - entityView.setEntityId(entityIdUUID != null ? new DeviceId(entityIdUUID) : null); - } else if (EdgeEntityType.ASSET.equals(entityViewUpdateMsg.getEntityType())) { - entityView.setEntityId(entityIdUUID != null ? new AssetId(entityIdUUID) : null); - } - return entityView; - } - - @Override - protected void setCustomerId(TenantId tenantId, CustomerId customerId, EntityView entityView, EntityViewUpdateMsg entityViewUpdateMsg) { - CustomerId customerUUID = safeGetCustomerId(entityViewUpdateMsg.getCustomerIdMSB(), entityViewUpdateMsg.getCustomerIdLSB()); - entityView.setCustomerId(customerUUID != null ? customerUUID : customerId); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorV2.java deleted file mode 100644 index fa4177552e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewProcessorV2.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.entityview; - -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Primary -@Component -@TbCoreComponent -public class EntityViewProcessorV2 extends EntityViewEdgeProcessor { - - @Override - protected EntityView constructEntityViewFromUpdateMsg(TenantId tenantId, EntityViewId entityViewId, EntityViewUpdateMsg entityViewUpdateMsg) { - return JacksonUtil.fromString(entityViewUpdateMsg.getEntity(), EntityView.class, true); - } - - @Override - protected void setCustomerId(TenantId tenantId, CustomerId customerId, EntityView entityView, EntityViewUpdateMsg entityViewUpdateMsg) { - CustomerId customerUUID = entityView.getCustomerId() != null ? entityView.getCustomerId() : customerId; - entityView.setCustomerId(customerUUID); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationEdgeProcessor.java deleted file mode 100644 index 67aa1dece2..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationEdgeProcessor.java +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.notification; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.EdgeUtils; -import org.thingsboard.server.common.data.edge.EdgeEvent; -import org.thingsboard.server.common.data.id.NotificationRuleId; -import org.thingsboard.server.common.data.id.NotificationTargetId; -import org.thingsboard.server.common.data.id.NotificationTemplateId; -import org.thingsboard.server.common.data.notification.rule.NotificationRule; -import org.thingsboard.server.common.data.notification.targets.NotificationTarget; -import org.thingsboard.server.common.data.notification.template.NotificationTemplate; -import org.thingsboard.server.dao.notification.NotificationRuleService; -import org.thingsboard.server.dao.notification.NotificationTargetService; -import org.thingsboard.server.dao.notification.NotificationTemplateService; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.NotificationRuleUpdateMsg; -import org.thingsboard.server.gen.edge.v1.NotificationTargetUpdateMsg; -import org.thingsboard.server.gen.edge.v1.NotificationTemplateUpdateMsg; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.notification.NotificationMsgConstructor; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; - -@Slf4j -@Component -@TbCoreComponent -public class NotificationEdgeProcessor extends BaseEdgeProcessor { - - @Autowired - protected NotificationRuleService notificationRuleService; - - @Autowired - private NotificationTargetService notificationTargetService; - - @Autowired - private NotificationTemplateService notificationTemplateService; - - @Autowired - private NotificationMsgConstructor notificationMsgConstructor; - - public DownlinkMsg convertNotificationRuleToDownlink(EdgeEvent edgeEvent) { - NotificationRuleId notificationRuleId = new NotificationRuleId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; - switch (edgeEvent.getAction()) { - case ADDED, UPDATED -> { - NotificationRule notificationRule = notificationRuleService.findNotificationRuleById(edgeEvent.getTenantId(), notificationRuleId); - if (notificationRule != null) { - UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - NotificationRuleUpdateMsg notificationRuleUpdateMsg = notificationMsgConstructor.constructNotificationRuleUpdateMsg(msgType, notificationRule); - downlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addNotificationRuleUpdateMsg(notificationRuleUpdateMsg) - .build(); - } - } - case DELETED -> { - NotificationRuleUpdateMsg notificationRuleUpdateMsg = notificationMsgConstructor.constructNotificationRuleDeleteMsg(notificationRuleId); - downlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addNotificationRuleUpdateMsg(notificationRuleUpdateMsg) - .build(); - } - } - return downlinkMsg; - } - - public DownlinkMsg convertNotificationTargetToDownlink(EdgeEvent edgeEvent) { - NotificationTargetId notificationTargetId = new NotificationTargetId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; - switch (edgeEvent.getAction()) { - case ADDED, UPDATED -> { - NotificationTarget notificationTarget = notificationTargetService.findNotificationTargetById(edgeEvent.getTenantId(), notificationTargetId); - if (notificationTarget != null) { - UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - NotificationTargetUpdateMsg notificationTargetUpdateMsg = notificationMsgConstructor.constructNotificationTargetUpdateMsg(msgType, notificationTarget); - downlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addNotificationTargetUpdateMsg(notificationTargetUpdateMsg) - .build(); - } - } - case DELETED -> { - NotificationTargetUpdateMsg notificationTargetUpdateMsg = notificationMsgConstructor.constructNotificationTargetDeleteMsg(notificationTargetId); - downlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addNotificationTargetUpdateMsg(notificationTargetUpdateMsg) - .build(); - } - } - return downlinkMsg; - } - - public DownlinkMsg convertNotificationTemplateToDownlink(EdgeEvent edgeEvent) { - NotificationTemplateId notificationTemplateId = new NotificationTemplateId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; - switch (edgeEvent.getAction()) { - case ADDED, UPDATED -> { - NotificationTemplate notificationTemplate = notificationTemplateService.findNotificationTemplateById(edgeEvent.getTenantId(), notificationTemplateId); - if (notificationTemplate != null) { - UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - NotificationTemplateUpdateMsg notificationTemplateUpdateMsg = notificationMsgConstructor.constructNotificationTemplateUpdateMsg(msgType, notificationTemplate); - downlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addNotificationTemplateUpdateMsg(notificationTemplateUpdateMsg) - .build(); - } - } - case DELETED -> { - NotificationTemplateUpdateMsg notificationTemplateUpdateMsg = notificationMsgConstructor.constructNotificationTemplateDeleteMsg(notificationTemplateId); - downlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addNotificationTemplateUpdateMsg(notificationTemplateUpdateMsg) - .build(); - } - } - return downlinkMsg; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationRuleEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationRuleEdgeProcessor.java new file mode 100644 index 0000000000..d5f68f5427 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationRuleEdgeProcessor.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.notification; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.NotificationRuleId; +import org.thingsboard.server.common.data.notification.rule.NotificationRule; +import org.thingsboard.server.dao.notification.NotificationRuleService; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.NotificationRuleUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +@Slf4j +@Component +@TbCoreComponent +public class NotificationRuleEdgeProcessor extends BaseEdgeProcessor { + + @Autowired + private NotificationRuleService notificationRuleService; + + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { + NotificationRuleId notificationRuleId = new NotificationRuleId(edgeEvent.getEntityId()); + DownlinkMsg downlinkMsg = null; + switch (edgeEvent.getAction()) { + case ADDED, UPDATED -> { + NotificationRule notificationRule = notificationRuleService.findNotificationRuleById(edgeEvent.getTenantId(), notificationRuleId); + if (notificationRule != null) { + UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + NotificationRuleUpdateMsg notificationRuleUpdateMsg = EdgeMsgConstructorUtils.constructNotificationRuleUpdateMsg(msgType, notificationRule); + downlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addNotificationRuleUpdateMsg(notificationRuleUpdateMsg) + .build(); + } + } + case DELETED -> { + NotificationRuleUpdateMsg notificationRuleUpdateMsg = EdgeMsgConstructorUtils.constructNotificationRuleDeleteMsg(notificationRuleId); + downlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addNotificationRuleUpdateMsg(notificationRuleUpdateMsg) + .build(); + } + } + return downlinkMsg; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTargetEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTargetEdgeProcessor.java new file mode 100644 index 0000000000..08d0efcaad --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTargetEdgeProcessor.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.notification; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.NotificationTargetId; +import org.thingsboard.server.common.data.notification.targets.NotificationTarget; +import org.thingsboard.server.dao.notification.NotificationTargetService; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.NotificationTargetUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +@Slf4j +@Component +@TbCoreComponent +public class NotificationTargetEdgeProcessor extends BaseEdgeProcessor { + + @Autowired + private NotificationTargetService notificationTargetService; + + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { + NotificationTargetId notificationTargetId = new NotificationTargetId(edgeEvent.getEntityId()); + DownlinkMsg downlinkMsg = null; + switch (edgeEvent.getAction()) { + case ADDED, UPDATED -> { + NotificationTarget notificationTarget = notificationTargetService.findNotificationTargetById(edgeEvent.getTenantId(), notificationTargetId); + if (notificationTarget != null) { + UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + NotificationTargetUpdateMsg notificationTargetUpdateMsg = EdgeMsgConstructorUtils.constructNotificationTargetUpdateMsg(msgType, notificationTarget); + downlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addNotificationTargetUpdateMsg(notificationTargetUpdateMsg) + .build(); + } + } + case DELETED -> { + NotificationTargetUpdateMsg notificationTargetUpdateMsg = EdgeMsgConstructorUtils.constructNotificationTargetDeleteMsg(notificationTargetId); + downlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addNotificationTargetUpdateMsg(notificationTargetUpdateMsg) + .build(); + } + } + return downlinkMsg; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTemplateEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTemplateEdgeProcessor.java new file mode 100644 index 0000000000..1a4227a5fc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTemplateEdgeProcessor.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.notification; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.NotificationTemplateId; +import org.thingsboard.server.common.data.notification.template.NotificationTemplate; +import org.thingsboard.server.dao.notification.NotificationTemplateService; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.NotificationTemplateUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +@Slf4j +@Component +@TbCoreComponent +public class NotificationTemplateEdgeProcessor extends BaseEdgeProcessor { + + @Autowired + private NotificationTemplateService notificationTemplateService; + + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { + NotificationTemplateId notificationTemplateId = new NotificationTemplateId(edgeEvent.getEntityId()); + DownlinkMsg downlinkMsg = null; + switch (edgeEvent.getAction()) { + case ADDED, UPDATED -> { + NotificationTemplate notificationTemplate = notificationTemplateService.findNotificationTemplateById(edgeEvent.getTenantId(), notificationTemplateId); + if (notificationTemplate != null) { + UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + NotificationTemplateUpdateMsg notificationTemplateUpdateMsg = EdgeMsgConstructorUtils.constructNotificationTemplateUpdateMsg(msgType, notificationTemplate); + downlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addNotificationTemplateUpdateMsg(notificationTemplateUpdateMsg) + .build(); + } + } + case DELETED -> { + NotificationTemplateUpdateMsg notificationTemplateUpdateMsg = EdgeMsgConstructorUtils.constructNotificationTemplateDeleteMsg(notificationTemplateId); + downlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addNotificationTemplateUpdateMsg(notificationTemplateUpdateMsg) + .build(); + } + } + return downlinkMsg; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2EdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/DomainEdgeProcessor.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2EdgeProcessor.java rename to application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/DomainEdgeProcessor.java index 9f4af7d0dc..a6016c97d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2EdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/DomainEdgeProcessor.java @@ -22,55 +22,46 @@ import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.domain.DomainInfo; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DomainId; -import org.thingsboard.server.common.data.id.OAuth2ClientId; import org.thingsboard.server.common.data.oauth2.OAuth2Client; import org.thingsboard.server.dao.domain.DomainService; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.OAuth2ClientUpdateMsg; import org.thingsboard.server.gen.edge.v1.OAuth2DomainUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.oauth2.OAuth2MsgConstructor; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.utils.EdgeVersionUtils; @Slf4j @Component @TbCoreComponent -public class OAuth2EdgeProcessor extends BaseEdgeProcessor { +public class DomainEdgeProcessor extends BaseEdgeProcessor { @Autowired private DomainService domainService; - @Autowired - private OAuth2MsgConstructor oAuth2MsgConstructor; - - public DownlinkMsg convertOAuth2DomainEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { - if (EdgeVersionUtils.isEdgeVersionOlderThan(edgeVersion, EdgeVersion.V_3_8_0)) { - return null; - } + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { DomainId domainId = new DomainId(edgeEvent.getEntityId()); - switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { DomainInfo domainInfo = domainService.findDomainInfoById(edgeEvent.getTenantId(), domainId); if (domainInfo != null && domainInfo.isPropagateToEdge()) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - OAuth2DomainUpdateMsg oAuth2DomainUpdateMsg = oAuth2MsgConstructor.constructOAuth2DomainUpdateMsg(msgType, domainInfo); + OAuth2DomainUpdateMsg oAuth2DomainUpdateMsg = EdgeMsgConstructorUtils.constructOAuth2DomainUpdateMsg(msgType, domainInfo); DownlinkMsg.Builder builder = DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addOAuth2DomainUpdateMsg(oAuth2DomainUpdateMsg); domainInfo.getOauth2ClientInfos().forEach(clientInfo -> { OAuth2Client oauth2Client = edgeCtx.getOAuth2ClientService().findOAuth2ClientById(edgeEvent.getTenantId(), clientInfo.getId()); - OAuth2ClientUpdateMsg oAuth2ClientUpdateMsg = oAuth2MsgConstructor.constructOAuth2ClientUpdateMsg(msgType, oauth2Client); + OAuth2ClientUpdateMsg oAuth2ClientUpdateMsg = EdgeMsgConstructorUtils.constructOAuth2ClientUpdateMsg(msgType, oauth2Client); builder.addOAuth2ClientUpdateMsg(oAuth2ClientUpdateMsg); }); return builder.build(); } } case DELETED -> { - OAuth2DomainUpdateMsg oAuth2DomainUpdateMsg = oAuth2MsgConstructor.constructOAuth2DomainDeleteMsg(domainId); + OAuth2DomainUpdateMsg oAuth2DomainUpdateMsg = EdgeMsgConstructorUtils.constructOAuth2DomainDeleteMsg(domainId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addOAuth2DomainUpdateMsg(oAuth2DomainUpdateMsg) @@ -80,37 +71,4 @@ public class OAuth2EdgeProcessor extends BaseEdgeProcessor { return null; } - public DownlinkMsg convertOAuth2ClientEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { - if (EdgeVersionUtils.isEdgeVersionOlderThan(edgeVersion, EdgeVersion.V_3_8_0)) { - return null; - } - OAuth2ClientId oAuth2ClientId = new OAuth2ClientId(edgeEvent.getEntityId()); - - switch (edgeEvent.getAction()) { - case ADDED, UPDATED -> { - boolean isPropagateToEdge = edgeCtx.getOAuth2ClientService().isPropagateOAuth2ClientToEdge(edgeEvent.getTenantId(), oAuth2ClientId); - if (!isPropagateToEdge) { - return null; - } - OAuth2Client oAuth2Client = edgeCtx.getOAuth2ClientService().findOAuth2ClientById(edgeEvent.getTenantId(), oAuth2ClientId); - if (oAuth2Client != null) { - UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - OAuth2ClientUpdateMsg oAuth2ClientUpdateMsg = oAuth2MsgConstructor.constructOAuth2ClientUpdateMsg(msgType, oAuth2Client); - return DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addOAuth2ClientUpdateMsg(oAuth2ClientUpdateMsg) - .build(); - } - } - case DELETED -> { - OAuth2ClientUpdateMsg oAuth2ClientDeleteMsg = oAuth2MsgConstructor.constructOAuth2ClientDeleteMsg(oAuth2ClientId); - return DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addOAuth2ClientUpdateMsg(oAuth2ClientDeleteMsg) - .build(); - } - } - return null; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2ClientEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2ClientEdgeProcessor.java new file mode 100644 index 0000000000..cc1c361824 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2ClientEdgeProcessor.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.oauth2.OAuth2Client; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.OAuth2ClientUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +@Slf4j +@Component +@TbCoreComponent +public class OAuth2ClientEdgeProcessor extends BaseEdgeProcessor { + + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { + OAuth2ClientId oAuth2ClientId = new OAuth2ClientId(edgeEvent.getEntityId()); + + switch (edgeEvent.getAction()) { + case ADDED, UPDATED -> { + boolean isPropagateToEdge = edgeCtx.getOAuth2ClientService().isPropagateOAuth2ClientToEdge(edgeEvent.getTenantId(), oAuth2ClientId); + if (!isPropagateToEdge) { + return null; + } + OAuth2Client oAuth2Client = edgeCtx.getOAuth2ClientService().findOAuth2ClientById(edgeEvent.getTenantId(), oAuth2ClientId); + if (oAuth2Client != null) { + UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + OAuth2ClientUpdateMsg oAuth2ClientUpdateMsg = EdgeMsgConstructorUtils.constructOAuth2ClientUpdateMsg(msgType, oAuth2Client); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addOAuth2ClientUpdateMsg(oAuth2ClientUpdateMsg) + .build(); + } + } + case DELETED -> { + OAuth2ClientUpdateMsg oAuth2ClientDeleteMsg = EdgeMsgConstructorUtils.constructOAuth2ClientDeleteMsg(oAuth2ClientId); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addOAuth2ClientUpdateMsg(oAuth2ClientDeleteMsg) + .build(); + } + } + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java index fc803caf4d..c09d735f5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java @@ -16,38 +16,32 @@ package org.thingsboard.server.service.edge.rpc.processor.ota; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.ota.OtaPackageMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.ota.OtaPackageMsgConstructorFactory; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; -@Component @Slf4j +@Component @TbCoreComponent public class OtaPackageEdgeProcessor extends BaseEdgeProcessor { - @Autowired - private OtaPackageMsgConstructorFactory otaPackageMsgConstructorFactory; - - public DownlinkMsg convertOtaPackageEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { OtaPackageId otaPackageId = new OtaPackageId(edgeEvent.getEntityId()); - var msgConstructor = (OtaPackageMsgConstructor) otaPackageMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { OtaPackage otaPackage = edgeCtx.getOtaPackageService().findOtaPackageById(edgeEvent.getTenantId(), otaPackageId); if (otaPackage != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - OtaPackageUpdateMsg otaPackageUpdateMsg = msgConstructor.constructOtaPackageUpdatedMsg(msgType, otaPackage); + OtaPackageUpdateMsg otaPackageUpdateMsg = EdgeMsgConstructorUtils.constructOtaPackageUpdatedMsg(msgType, otaPackage); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addOtaPackageUpdateMsg(otaPackageUpdateMsg) @@ -55,7 +49,7 @@ public class OtaPackageEdgeProcessor extends BaseEdgeProcessor { } } case DELETED -> { - OtaPackageUpdateMsg otaPackageUpdateMsg = msgConstructor.constructOtaPackageDeleteMsg(otaPackageId); + OtaPackageUpdateMsg otaPackageUpdateMsg = EdgeMsgConstructorUtils.constructOtaPackageDeleteMsg(otaPackageId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addOtaPackageUpdateMsg(otaPackageUpdateMsg) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java index 76f068ac7a..937049f447 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java @@ -16,19 +16,16 @@ package org.thingsboard.server.service.edge.rpc.processor.queue; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.queue.QueueMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.queue.QueueMsgConstructorFactory; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -36,18 +33,15 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class QueueEdgeProcessor extends BaseEdgeProcessor { - @Autowired - private QueueMsgConstructorFactory queueMsgConstructorFactory; - - public DownlinkMsg convertQueueEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { QueueId queueId = new QueueId(edgeEvent.getEntityId()); - var msgConstructor = (QueueMsgConstructor) queueMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { Queue queue = edgeCtx.getQueueService().findQueueById(edgeEvent.getTenantId(), queueId); if (queue != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - QueueUpdateMsg queueUpdateMsg = msgConstructor.constructQueueUpdatedMsg(msgType, queue); + QueueUpdateMsg queueUpdateMsg = EdgeMsgConstructorUtils.constructQueueUpdatedMsg(msgType, queue); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addQueueUpdateMsg(queueUpdateMsg) @@ -55,7 +49,7 @@ public class QueueEdgeProcessor extends BaseEdgeProcessor { } } case DELETED -> { - QueueUpdateMsg queueDeleteMsg = msgConstructor.constructQueueDeleteMsg(queueId); + QueueUpdateMsg queueDeleteMsg = EdgeMsgConstructorUtils.constructQueueDeleteMsg(queueId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addQueueUpdateMsg(queueDeleteMsg) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java index 0c6da73a25..488e065a26 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.relation; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; @@ -29,7 +30,7 @@ public abstract class BaseRelationProcessor extends BaseEdgeProcessor { protected ListenableFuture processRelationMsg(TenantId tenantId, RelationUpdateMsg relationUpdateMsg) { log.trace("[{}] processRelationMsg [{}]", tenantId, relationUpdateMsg); try { - EntityRelation entityRelation = constructEntityRelationFromUpdateMsg(relationUpdateMsg); + EntityRelation entityRelation = JacksonUtil.fromString(relationUpdateMsg.getEntity(), EntityRelation.class, true); if (entityRelation == null) { throw new RuntimeException("[{" + tenantId + "}] relationUpdateMsg {" + relationUpdateMsg + "} cannot be converted to entity relation"); } @@ -56,6 +57,4 @@ public abstract class BaseRelationProcessor extends BaseEdgeProcessor { return Futures.immediateFuture(null); } - protected abstract EntityRelation constructEntityRelationFromUpdateMsg(RelationUpdateMsg relationUpdateMsg); - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java index 87b9797620..bc9b955033 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.relation; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityType; @@ -30,12 +30,11 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.service.edge.rpc.constructor.relation.RelationMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.relation.RelationMsgConstructorFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import java.util.ArrayList; import java.util.HashSet; @@ -43,10 +42,9 @@ import java.util.List; import java.util.Set; @Slf4j -public abstract class RelationEdgeProcessor extends BaseRelationProcessor implements RelationProcessor { - - @Autowired - private RelationMsgConstructorFactory relationMsgConstructorFactory; +@Component +@TbCoreComponent +public class RelationEdgeProcessor extends BaseRelationProcessor implements RelationProcessor { @Override public ListenableFuture processRelationMsgFromEdge(TenantId tenantId, Edge edge, RelationUpdateMsg relationUpdateMsg) { @@ -60,18 +58,7 @@ public abstract class RelationEdgeProcessor extends BaseRelationProcessor implem } @Override - public DownlinkMsg convertRelationEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { - EntityRelation entityRelation = JacksonUtil.convertValue(edgeEvent.getBody(), EntityRelation.class); - UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - RelationUpdateMsg relationUpdateMsg = ((RelationMsgConstructor) relationMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)) - .constructRelationUpdatedMsg(msgType, entityRelation); - return DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addRelationUpdateMsg(relationUpdateMsg) - .build(); - } - - public ListenableFuture processRelationNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { + public ListenableFuture processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { EntityRelation relation = JacksonUtil.fromString(edgeNotificationMsg.getBody(), EntityRelation.class); if (relation == null || (relation.getFrom().getEntityType().equals(EntityType.EDGE) || relation.getTo().getEntityType().equals(EntityType.EDGE))) { return Futures.immediateFuture(null); @@ -97,4 +84,15 @@ public abstract class RelationEdgeProcessor extends BaseRelationProcessor implem return Futures.transform(Futures.allAsList(futures), voids -> null, dbCallbackExecutorService); } + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { + EntityRelation entityRelation = JacksonUtil.convertValue(edgeEvent.getBody(), EntityRelation.class); + UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + RelationUpdateMsg relationUpdateMsg = EdgeMsgConstructorUtils.constructRelationUpdatedMsg(msgType, entityRelation); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addRelationUpdateMsg(relationUpdateMsg) + .build(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorFactory.java deleted file mode 100644 index 3dea55205e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.relation; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessorFactory; - -@Component -@TbCoreComponent -public class RelationEdgeProcessorFactory extends BaseEdgeProcessorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorV1.java deleted file mode 100644 index 647cec120a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorV1.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.relation; - -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.UUID; - -@Component -@TbCoreComponent -public class RelationEdgeProcessorV1 extends RelationEdgeProcessor { - - @Override - protected EntityRelation constructEntityRelationFromUpdateMsg(RelationUpdateMsg relationUpdateMsg) { - EntityRelation entityRelation = new EntityRelation(); - - UUID fromUUID = new UUID(relationUpdateMsg.getFromIdMSB(), relationUpdateMsg.getFromIdLSB()); - EntityId fromId = EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(relationUpdateMsg.getFromEntityType()), fromUUID); - entityRelation.setFrom(fromId); - - UUID toUUID = new UUID(relationUpdateMsg.getToIdMSB(), relationUpdateMsg.getToIdLSB()); - EntityId toId = EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(relationUpdateMsg.getToEntityType()), toUUID); - entityRelation.setTo(toId); - - entityRelation.setType(relationUpdateMsg.getType()); - entityRelation.setTypeGroup(relationUpdateMsg.hasTypeGroup() - ? RelationTypeGroup.valueOf(relationUpdateMsg.getTypeGroup()) : RelationTypeGroup.COMMON); - entityRelation.setAdditionalInfo(JacksonUtil.toJsonNode(relationUpdateMsg.getAdditionalInfo())); - return entityRelation; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorV2.java deleted file mode 100644 index 7b31da421d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessorV2.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.relation; - -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Primary -@Component -@TbCoreComponent -public class RelationEdgeProcessorV2 extends RelationEdgeProcessor { - - @Override - protected EntityRelation constructEntityRelationFromUpdateMsg(RelationUpdateMsg relationUpdateMsg) { - return JacksonUtil.fromString(relationUpdateMsg.getEntity(), EntityRelation.class, true); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationProcessor.java index 82a893e9ba..d3d31cd780 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationProcessor.java @@ -17,10 +17,7 @@ package org.thingsboard.server.service.edge.rpc.processor.relation; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; @@ -28,6 +25,4 @@ public interface RelationProcessor extends EdgeProcessor { ListenableFuture processRelationMsgFromEdge(TenantId tenantId, Edge edge, RelationUpdateMsg relationUpdateMsg); - DownlinkMsg convertRelationEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java index ac7bd6768c..11a0d70211 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.resource; import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbResource; @@ -38,7 +39,7 @@ public abstract class BaseResourceProcessor extends BaseEdgeProcessor { protected boolean saveOrUpdateTbResource(TenantId tenantId, TbResourceId tbResourceId, ResourceUpdateMsg resourceUpdateMsg) { boolean resourceKeyUpdated = false; try { - TbResource resource = constructResourceFromUpdateMsg(tenantId, tbResourceId, resourceUpdateMsg); + TbResource resource = JacksonUtil.fromString(resourceUpdateMsg.getEntity(), TbResource.class, true); if (resource == null) { throw new RuntimeException("[{" + tenantId + "}] resourceUpdateMsg {" + resourceUpdateMsg + " } cannot be converted to resource"); } @@ -76,6 +77,4 @@ public abstract class BaseResourceProcessor extends BaseEdgeProcessor { return resourceKeyUpdated; } - protected abstract TbResource constructResourceFromUpdateMsg(TenantId tenantId, TbResourceId tbResourceId, ResourceUpdateMsg resourceUpdateMsg); - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java index 10c58e89d0..1d5ebc188f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.resource; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.edge.Edge; @@ -29,19 +29,17 @@ import org.thingsboard.server.common.data.id.TbResourceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.resource.ResourceMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.resource.ResourceMsgConstructorFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import java.util.UUID; @Slf4j -public abstract class ResourceEdgeProcessor extends BaseResourceProcessor implements ResourceProcessor { - - @Autowired - private ResourceMsgConstructorFactory resourceMsgConstructorFactory; +@Component +@TbCoreComponent +public class ResourceEdgeProcessor extends BaseResourceProcessor implements ResourceProcessor { @Override public ListenableFuture processResourceMsgFromEdge(TenantId tenantId, Edge edge, ResourceUpdateMsg resourceUpdateMsg) { @@ -75,23 +73,22 @@ public abstract class ResourceEdgeProcessor extends BaseResourceProcessor implem } @Override - public DownlinkMsg convertResourceEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { TbResourceId tbResourceId = new TbResourceId(edgeEvent.getEntityId()); - var msgConstructor = (ResourceMsgConstructor) resourceMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { TbResource tbResource = edgeCtx.getResourceService().findResourceById(edgeEvent.getTenantId(), tbResourceId); if (tbResource != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - ResourceUpdateMsg resourceUpdateMsg = msgConstructor.constructResourceUpdatedMsg(msgType, tbResource); - return resourceUpdateMsg != null ? DownlinkMsg.newBuilder() + ResourceUpdateMsg resourceUpdateMsg = EdgeMsgConstructorUtils.constructResourceUpdatedMsg(msgType, tbResource); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addResourceUpdateMsg(resourceUpdateMsg) - .build() : null; + .build(); } } case DELETED -> { - ResourceUpdateMsg resourceUpdateMsg = msgConstructor.constructResourceDeleteMsg(tbResourceId); + ResourceUpdateMsg resourceUpdateMsg = EdgeMsgConstructorUtils.constructResourceDeleteMsg(tbResourceId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addResourceUpdateMsg(resourceUpdateMsg) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorFactory.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorFactory.java deleted file mode 100644 index 9f45aacdeb..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.resource; - -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessorFactory; - -@Component -@TbCoreComponent -public class ResourceEdgeProcessorFactory extends BaseEdgeProcessorFactory { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV1.java deleted file mode 100644 index f272d447a4..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV1.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.resource; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.ResourceType; -import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.id.TbResourceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Component -@TbCoreComponent -public class ResourceEdgeProcessorV1 extends ResourceEdgeProcessor { - - @Override - protected TbResource constructResourceFromUpdateMsg(TenantId tenantId, TbResourceId tbResourceId, ResourceUpdateMsg resourceUpdateMsg) { - TbResource resource = new TbResource(); - if (resourceUpdateMsg.getIsSystem()) { - resource.setTenantId(TenantId.SYS_TENANT_ID); - } else { - resource.setTenantId(tenantId); - } - resource.setCreatedTime(Uuids.unixTimestamp(tbResourceId.getId())); - resource.setTitle(resourceUpdateMsg.getTitle()); - resource.setResourceKey(resourceUpdateMsg.getResourceKey()); - resource.setResourceType(ResourceType.valueOf(resourceUpdateMsg.getResourceType())); - resource.setFileName(resourceUpdateMsg.getFileName()); - resource.setEncodedData(resourceUpdateMsg.hasData() ? resourceUpdateMsg.getData() : null); - resource.setEtag(resourceUpdateMsg.hasEtag() ? resourceUpdateMsg.getEtag() : null); - return resource; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV2.java deleted file mode 100644 index 108a6c8ecb..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV2.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor.resource; - -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.id.TbResourceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Primary -@Component -@TbCoreComponent -public class ResourceEdgeProcessorV2 extends ResourceEdgeProcessor { - - @Override - protected TbResource constructResourceFromUpdateMsg(TenantId tenantId, TbResourceId tbResourceId, ResourceUpdateMsg resourceUpdateMsg) { - return JacksonUtil.fromString(resourceUpdateMsg.getEntity(), TbResource.class, true); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceProcessor.java index a52d31a98b..5ed1f7c363 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceProcessor.java @@ -17,10 +17,7 @@ package org.thingsboard.server.service.edge.rpc.processor.resource; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; @@ -28,6 +25,4 @@ public interface ResourceProcessor extends EdgeProcessor { ListenableFuture processResourceMsgFromEdge(TenantId tenantId, Edge edge, ResourceUpdateMsg resourceUpdateMsg); - DownlinkMsg convertResourceEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java index 6df069e400..72d6565b92 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.edge.rpc.processor.rule; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.Edge; @@ -25,13 +24,11 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.rule.RuleChainMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.rule.RuleChainMsgConstructorFactory; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; import static org.thingsboard.server.dao.edge.EdgeServiceImpl.EDGE_IS_ROOT_BODY_KEY; @@ -41,13 +38,10 @@ import static org.thingsboard.server.dao.edge.EdgeServiceImpl.EDGE_IS_ROOT_BODY_ @TbCoreComponent public class RuleChainEdgeProcessor extends BaseEdgeProcessor { - @Autowired - private RuleChainMsgConstructorFactory ruleChainMsgConstructorFactory; - - public DownlinkMsg convertRuleChainEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { RuleChainId ruleChainId = new RuleChainId(edgeEvent.getEntityId()); DownlinkMsg downlinkMsg = null; - var msgConstructor = (RuleChainMsgConstructor) ruleChainMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED, ASSIGNED_TO_EDGE -> { RuleChain ruleChain = edgeCtx.getRuleChainService().findRuleChainById(edgeEvent.getTenantId(), ruleChainId); @@ -64,46 +58,25 @@ public class RuleChainEdgeProcessor extends BaseEdgeProcessor { isRoot = edge.getRootRuleChainId().equals(ruleChainId); } UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - RuleChainUpdateMsg ruleChainUpdateMsg = msgConstructor.constructRuleChainUpdatedMsg(msgType, ruleChain, isRoot); + RuleChainUpdateMsg ruleChainUpdateMsg = EdgeMsgConstructorUtils.constructRuleChainUpdatedMsg(msgType, ruleChain, isRoot); DownlinkMsg.Builder builder = DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addRuleChainUpdateMsg(ruleChainUpdateMsg); RuleChainMetaData ruleChainMetaData = edgeCtx.getRuleChainService().loadRuleChainMetaData(edgeEvent.getTenantId(), ruleChainId); - RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = msgConstructor - .constructRuleChainMetadataUpdatedMsg(edgeEvent.getTenantId(), msgType, ruleChainMetaData, edgeVersion); - if (ruleChainMetadataUpdateMsg != null) { - builder.addRuleChainMetadataUpdateMsg(ruleChainMetadataUpdateMsg); - } + RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = EdgeMsgConstructorUtils + .constructRuleChainMetadataUpdatedMsg(msgType, ruleChainMetaData); + builder.addRuleChainMetadataUpdateMsg(ruleChainMetadataUpdateMsg); downlinkMsg = builder.build(); } } case DELETED, UNASSIGNED_FROM_EDGE -> downlinkMsg = DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addRuleChainUpdateMsg(msgConstructor.constructRuleChainDeleteMsg(ruleChainId)) + .addRuleChainUpdateMsg(EdgeMsgConstructorUtils.constructRuleChainDeleteMsg(ruleChainId)) .build(); } return downlinkMsg; } - public DownlinkMsg convertRuleChainMetadataEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { - RuleChainId ruleChainId = new RuleChainId(edgeEvent.getEntityId()); - RuleChain ruleChain = edgeCtx.getRuleChainService().findRuleChainById(edgeEvent.getTenantId(), ruleChainId); - var msgConstructor = (RuleChainMsgConstructor) ruleChainMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); - if (ruleChain != null) { - RuleChainMetaData ruleChainMetaData = edgeCtx.getRuleChainService().loadRuleChainMetaData(edgeEvent.getTenantId(), ruleChainId); - UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = msgConstructor - .constructRuleChainMetadataUpdatedMsg(edgeEvent.getTenantId(), msgType, ruleChainMetaData, edgeVersion); - if (ruleChainMetadataUpdateMsg != null) { - return DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addRuleChainMetadataUpdateMsg(ruleChainMetadataUpdateMsg) - .build(); - } - } - return null; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainMetadataEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainMetadataEdgeProcessor.java new file mode 100644 index 0000000000..eedfc835ae --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainMetadataEdgeProcessor.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.rule; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +@Slf4j +@Component +@TbCoreComponent +public class RuleChainMetadataEdgeProcessor extends BaseEdgeProcessor { + + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { + RuleChainId ruleChainId = new RuleChainId(edgeEvent.getEntityId()); + RuleChain ruleChain = edgeCtx.getRuleChainService().findRuleChainById(edgeEvent.getTenantId(), ruleChainId); + if (ruleChain != null) { + RuleChainMetaData ruleChainMetaData = edgeCtx.getRuleChainService().loadRuleChainMetaData(edgeEvent.getTenantId(), ruleChainId); + UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = EdgeMsgConstructorUtils + .constructRuleChainMetadataUpdatedMsg(msgType, ruleChainMetaData); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addRuleChainMetadataUpdateMsg(ruleChainMetadataUpdateMsg) + .build(); + } + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java index 3c0d4fb99c..bdfa597ae7 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.edge.rpc.processor.settings; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; @@ -24,10 +23,7 @@ import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.gen.edge.v1.AdminSettingsUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.settings.AdminSettingsMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.settings.AdminSettingsMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -35,19 +31,16 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class AdminSettingsEdgeProcessor extends BaseEdgeProcessor { - @Autowired - private AdminSettingsMsgConstructorFactory adminSettingsMsgConstructorFactory; - - public DownlinkMsg convertAdminSettingsEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { AdminSettings adminSettings = JacksonUtil.convertValue(edgeEvent.getBody(), AdminSettings.class); if (adminSettings == null) { return null; } - AdminSettingsUpdateMsg adminSettingsUpdateMsg = ((AdminSettingsMsgConstructor) - adminSettingsMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructAdminSettingsUpdateMsg(adminSettings); + AdminSettingsUpdateMsg msg = AdminSettingsUpdateMsg.newBuilder().setEntity(JacksonUtil.toString(adminSettings)).build(); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addAdminSettingsUpdateMsg(adminSettingsUpdateMsg) + .addAdminSettingsUpdateMsg(msg) .build(); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java index d94d4d8939..1f426c9982 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java @@ -70,7 +70,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.service.edge.rpc.constructor.telemetry.EntityDataMsgConstructor; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -83,9 +83,6 @@ import java.util.UUID; @Slf4j public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { - @Autowired - private EntityDataMsgConstructor entityDataMsgConstructor; - @Autowired private PartitionService partitionService; @@ -356,8 +353,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { } } String bodyJackson = JacksonUtil.toString(body); - return bodyJackson == null ? null : - entityDataMsgConstructor.constructEntityDataMsg(tenantId, entityId, actionType, JsonParser.parseString(bodyJackson)); + return bodyJackson == null ? null : EdgeMsgConstructorUtils.constructEntityDataMsg(tenantId, entityId, actionType, JsonParser.parseString(bodyJackson)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java index aa6795b123..311975501d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.edge.rpc.processor.tenant; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.Tenant; @@ -25,13 +24,11 @@ import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.tenant.TenantMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.tenant.TenantMsgConstructorFactory; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -39,19 +36,16 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class TenantEdgeProcessor extends BaseEdgeProcessor { - @Autowired - private TenantMsgConstructorFactory tenantMsgConstructorFactory; - - public DownlinkMsg convertTenantEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { TenantId tenantId = TenantId.fromUUID(edgeEvent.getEntityId()); - var msgConstructor = ((TenantMsgConstructor) tenantMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)); if (EdgeEventActionType.UPDATED.equals(edgeEvent.getAction())) { Tenant tenant = edgeCtx.getTenantService().findTenantById(tenantId); if (tenant != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - TenantUpdateMsg tenantUpdateMsg = msgConstructor.constructTenantUpdateMsg(msgType, tenant); + TenantUpdateMsg tenantUpdateMsg = EdgeMsgConstructorUtils.constructTenantUpdateMsg(msgType, tenant); TenantProfile tenantProfile = edgeCtx.getTenantProfileService().findTenantProfileById(tenantId, tenant.getTenantProfileId()); - TenantProfileUpdateMsg tenantProfileUpdateMsg = msgConstructor.constructTenantProfileUpdateMsg(msgType, tenantProfile, edgeVersion); + TenantProfileUpdateMsg tenantProfileUpdateMsg = EdgeMsgConstructorUtils.constructTenantProfileUpdateMsg(msgType, tenantProfile); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addTenantUpdateMsg(tenantUpdateMsg) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java index 07ba6039af..b02a8e2ce3 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.edge.rpc.processor.tenant; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.TenantProfile; @@ -24,12 +23,10 @@ import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.tenant.TenantMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.tenant.TenantMsgConstructorFactory; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -37,17 +34,14 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class TenantProfileEdgeProcessor extends BaseEdgeProcessor { - @Autowired - private TenantMsgConstructorFactory tenantMsgConstructorFactory; - - public DownlinkMsg convertTenantProfileEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { TenantProfileId tenantProfileId = new TenantProfileId(edgeEvent.getEntityId()); - var msgConstructor = ((TenantMsgConstructor) tenantMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)); if (EdgeEventActionType.UPDATED.equals(edgeEvent.getAction())) { TenantProfile tenantProfile = edgeCtx.getTenantProfileService().findTenantProfileById(edgeEvent.getTenantId(), tenantProfileId); if (tenantProfile != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - TenantProfileUpdateMsg tenantProfileUpdateMsg = msgConstructor.constructTenantProfileUpdateMsg(msgType, tenantProfile, edgeVersion); + TenantProfileUpdateMsg tenantProfileUpdateMsg = EdgeMsgConstructorUtils.constructTenantProfileUpdateMsg(msgType, tenantProfile); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addTenantProfileUpdateMsg(tenantProfileUpdateMsg) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java index 319812a33b..1d2f1f364a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.edge.rpc.processor.user; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.User; @@ -24,12 +23,10 @@ import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.user.UserMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.user.UserMsgConstructorFactory; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -37,12 +34,9 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class UserEdgeProcessor extends BaseEdgeProcessor { - @Autowired - private UserMsgConstructorFactory userMsgConstructorFactory; - - public DownlinkMsg convertUserEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { UserId userId = new UserId(edgeEvent.getEntityId()); - var msgConstructor = (UserMsgConstructor) userMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { User user = edgeCtx.getUserService().findUserById(edgeEvent.getTenantId(), userId); @@ -50,10 +44,10 @@ public class UserEdgeProcessor extends BaseEdgeProcessor { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); DownlinkMsg.Builder builder = DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addUserUpdateMsg(msgConstructor.constructUserUpdatedMsg(msgType, user)); + .addUserUpdateMsg(EdgeMsgConstructorUtils.constructUserUpdatedMsg(msgType, user)); UserCredentials userCredentialsByUserId = edgeCtx.getUserService().findUserCredentialsByUserId(edgeEvent.getTenantId(), userId); if (userCredentialsByUserId != null && userCredentialsByUserId.isEnabled()) { - builder.addUserCredentialsUpdateMsg(msgConstructor.constructUserCredentialsUpdatedMsg(userCredentialsByUserId)); + builder.addUserCredentialsUpdateMsg(EdgeMsgConstructorUtils.constructUserCredentialsUpdatedMsg(userCredentialsByUserId)); } return builder.build(); } @@ -61,13 +55,13 @@ public class UserEdgeProcessor extends BaseEdgeProcessor { case DELETED -> { return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addUserUpdateMsg(msgConstructor.constructUserDeleteMsg(userId)) + .addUserUpdateMsg(EdgeMsgConstructorUtils.constructUserDeleteMsg(userId)) .build(); } case CREDENTIALS_UPDATED -> { UserCredentials userCredentialsByUserId = edgeCtx.getUserService().findUserCredentialsByUserId(edgeEvent.getTenantId(), userId); if (userCredentialsByUserId != null && userCredentialsByUserId.isEnabled()) { - UserCredentialsUpdateMsg userCredentialsUpdateMsg = msgConstructor.constructUserCredentialsUpdatedMsg(userCredentialsByUserId); + UserCredentialsUpdateMsg userCredentialsUpdateMsg = EdgeMsgConstructorUtils.constructUserCredentialsUpdatedMsg(userCredentialsByUserId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addUserCredentialsUpdateMsg(userCredentialsUpdateMsg) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java index 87eece2841..5ea5817639 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java @@ -16,19 +16,16 @@ package org.thingsboard.server.service.edge.rpc.processor.widget; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.WidgetsBundleUpdateMsg; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.widget.WidgetMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.widget.WidgetMsgConstructorFactory; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; import java.util.List; @@ -38,19 +35,16 @@ import java.util.List; @TbCoreComponent public class WidgetBundleEdgeProcessor extends BaseEdgeProcessor { - @Autowired - private WidgetMsgConstructorFactory widgetMsgConstructorFactory; - - public DownlinkMsg convertWidgetsBundleEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { WidgetsBundleId widgetsBundleId = new WidgetsBundleId(edgeEvent.getEntityId()); - var msgConstructor = ((WidgetMsgConstructor) widgetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { WidgetsBundle widgetsBundle = edgeCtx.getWidgetsBundleService().findWidgetsBundleById(edgeEvent.getTenantId(), widgetsBundleId); if (widgetsBundle != null) { List widgets = edgeCtx.getWidgetTypeService().findWidgetFqnsByWidgetsBundleId(edgeEvent.getTenantId(), widgetsBundleId); UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - WidgetsBundleUpdateMsg widgetsBundleUpdateMsg = msgConstructor.constructWidgetsBundleUpdateMsg(msgType, widgetsBundle, widgets); + WidgetsBundleUpdateMsg widgetsBundleUpdateMsg = EdgeMsgConstructorUtils.constructWidgetsBundleUpdateMsg(msgType, widgetsBundle, widgets); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addWidgetsBundleUpdateMsg(widgetsBundleUpdateMsg) @@ -58,7 +52,7 @@ public class WidgetBundleEdgeProcessor extends BaseEdgeProcessor { } } case DELETED -> { - WidgetsBundleUpdateMsg widgetsBundleUpdateMsg = msgConstructor.constructWidgetsBundleDeleteMsg(widgetsBundleId); + WidgetsBundleUpdateMsg widgetsBundleUpdateMsg = EdgeMsgConstructorUtils.constructWidgetsBundleDeleteMsg(widgetsBundleId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addWidgetsBundleUpdateMsg(widgetsBundleUpdateMsg) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java index 10d2ce4e82..f3ea3ec1ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java @@ -16,19 +16,16 @@ package org.thingsboard.server.service.edge.rpc.processor.widget; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.constructor.widget.WidgetMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.widget.WidgetMsgConstructorFactory; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -36,18 +33,15 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class WidgetTypeEdgeProcessor extends BaseEdgeProcessor { - @Autowired - private WidgetMsgConstructorFactory widgetMsgConstructorFactory; - - public DownlinkMsg convertWidgetTypeEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { WidgetTypeId widgetTypeId = new WidgetTypeId(edgeEvent.getEntityId()); - var msgConstructor = (WidgetMsgConstructor) widgetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { WidgetTypeDetails widgetTypeDetails = edgeCtx.getWidgetTypeService().findWidgetTypeDetailsById(edgeEvent.getTenantId(), widgetTypeId); if (widgetTypeDetails != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - WidgetTypeUpdateMsg widgetTypeUpdateMsg = msgConstructor.constructWidgetTypeUpdateMsg(msgType, widgetTypeDetails, edgeVersion); + WidgetTypeUpdateMsg widgetTypeUpdateMsg = EdgeMsgConstructorUtils.constructWidgetTypeUpdateMsg(msgType, widgetTypeDetails); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addWidgetTypeUpdateMsg(widgetTypeUpdateMsg) @@ -55,7 +49,7 @@ public class WidgetTypeEdgeProcessor extends BaseEdgeProcessor { } } case DELETED -> { - WidgetTypeUpdateMsg widgetTypeUpdateMsg = msgConstructor.constructWidgetTypeDeleteMsg(widgetTypeId); + WidgetTypeUpdateMsg widgetTypeUpdateMsg = EdgeMsgConstructorUtils.constructWidgetTypeDeleteMsg(widgetTypeId); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addWidgetTypeUpdateMsg(widgetTypeUpdateMsg) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java index f219d7ae69..68df0c5424 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java @@ -247,36 +247,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService future; - switch (type) { - case EDGE -> future = edgeCtx.getEdgeProcessor().processEdgeNotification(tenantId, edgeNotificationMsg); - case ASSET -> future = edgeCtx.getAssetProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case ASSET_PROFILE -> future = edgeCtx.getAssetProfileProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case DEVICE -> future = edgeCtx.getDeviceProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case DEVICE_PROFILE -> future = edgeCtx.getDeviceProfileProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case ENTITY_VIEW -> future = edgeCtx.getEntityViewProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case DASHBOARD -> future = edgeCtx.getDashboardProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case RULE_CHAIN -> future = edgeCtx.getRuleChainProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case USER -> future = edgeCtx.getUserProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case CUSTOMER -> future = edgeCtx.getCustomerProcessor().processCustomerNotification(tenantId, edgeNotificationMsg); - case OTA_PACKAGE -> future = edgeCtx.getOtaPackageProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case WIDGETS_BUNDLE -> future = edgeCtx.getWidgetBundleProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case WIDGET_TYPE -> future = edgeCtx.getWidgetTypeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case QUEUE -> future = edgeCtx.getQueueProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case ALARM -> future = edgeCtx.getAlarmProcessor().processAlarmNotification(tenantId, edgeNotificationMsg); - case ALARM_COMMENT -> future = edgeCtx.getAlarmProcessor().processAlarmCommentNotification(tenantId, edgeNotificationMsg); - case RELATION -> future = edgeCtx.getRelationProcessor().processRelationNotification(tenantId, edgeNotificationMsg); - case TENANT -> future = edgeCtx.getTenantProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case TENANT_PROFILE -> future = edgeCtx.getTenantProfileProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case NOTIFICATION_RULE, NOTIFICATION_TARGET, NOTIFICATION_TEMPLATE -> - future = edgeCtx.getNotificationEdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case TB_RESOURCE -> future = edgeCtx.getResourceProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case DOMAIN, OAUTH2_CLIENT -> future = edgeCtx.getOAuth2EdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - default -> { - future = Futures.immediateFuture(null); - log.warn("[{}] Edge event type [{}] is not designed to be pushed to edge", tenantId, type); - } - } + ListenableFuture future = edgeCtx.getProcessor(type).processEntityNotification(tenantId, edgeNotificationMsg); Futures.addCallback(future, new FutureCallback<>() { @Override public void onSuccess(@Nullable Void unused) { diff --git a/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java b/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java index fcef33b968..3936b3ee63 100644 --- a/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java +++ b/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java @@ -136,7 +136,7 @@ public class EdgeGrpcClient implements EdgeRpcClient { .setConnectRequestMsg(ConnectRequestMsg.newBuilder() .setEdgeRoutingKey(edgeKey) .setEdgeSecret(edgeSecret) - .setEdgeVersion(EdgeVersion.V_3_9_0) + .setEdgeVersion(EdgeVersion.V_4_0_0) .setMaxInboundMessageSize(maxInboundMessageSize) .build()) .build()); diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 4dc762e14c..79d80fc35f 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -41,6 +41,7 @@ enum EdgeVersion { V_3_7_0 = 7; V_3_8_0 = 8; V_3_9_0 = 9; + V_4_0_0 = 10; } /** @@ -92,12 +93,10 @@ message ConnectResponseMsg { } message SyncRequestMsg { - bool syncRequired = 1 [deprecated = true]; optional bool fullSync = 2; } -message SyncCompletedMsg { -} +message SyncCompletedMsg {} message EdgeConfiguration { int64 edgeIdMSB = 1; @@ -120,7 +119,7 @@ enum UpdateMsgType { ENTITY_DELETED_RPC_MESSAGE = 2; ALARM_ACK_RPC_MESSAGE = 3; ALARM_CLEAR_RPC_MESSAGE = 4; - ENTITY_MERGE_RPC_MESSAGE = 5 [deprecated = true]; + // use 6 as a next number } message EntityDataProto { @@ -143,68 +142,18 @@ message RuleChainUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - string name = 4 [deprecated = true]; - optional int64 firstRuleNodeIdMSB = 5 [deprecated = true]; - optional int64 firstRuleNodeIdLSB = 6 [deprecated = true]; - bool root = 7 [deprecated = true]; - bool debugMode = 8 [deprecated = true]; - string configuration = 9 [deprecated = true]; string entity = 10; } message RuleChainMetadataUpdateMsg { UpdateMsgType msgType = 1; - int64 ruleChainIdMSB = 2 [deprecated = true]; - int64 ruleChainIdLSB = 3 [deprecated = true]; - int32 firstNodeIndex = 4 [deprecated = true]; - repeated RuleNodeProto nodes = 5 [deprecated = true]; - repeated NodeConnectionInfoProto connections = 6 [deprecated = true]; - repeated RuleChainConnectionInfoProto ruleChainConnections = 7 [deprecated = true]; string entity = 8; } -message RuleNodeProto { - option deprecated = true; - int64 idMSB = 1; - int64 idLSB = 2; - string type = 3; - string name = 4; - bool debugMode = 5 [deprecated = true]; - string configuration = 6; - string additionalInfo = 7; - bool singletonMode = 8; - int32 configurationVersion = 9; - string debugSettings = 10; -} - -message NodeConnectionInfoProto { - option deprecated = true; - int32 fromIndex = 1; - int32 toIndex = 2; - string type = 3; -} - -message RuleChainConnectionInfoProto { - option deprecated = true; - int32 fromIndex = 1; - int64 targetRuleChainIdMSB = 2; - int64 targetRuleChainIdLSB = 3; - string type = 4; - string additionalInfo = 5; -} - message DashboardUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - optional int64 customerIdMSB = 4 [deprecated = true]; - optional int64 customerIdLSB = 5 [deprecated = true]; - string title = 6 [deprecated = true]; - string configuration = 7 [deprecated = true]; - optional string assignedCustomers = 8 [deprecated = true]; - optional string image = 9 [deprecated = true]; - optional int32 mobileOrder = 10 [deprecated = true]; - bool mobileHide = 11 [deprecated = true]; string entity = 12; } @@ -212,20 +161,6 @@ message DeviceUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - optional int64 customerIdMSB = 4 [deprecated = true]; - optional int64 customerIdLSB = 5 [deprecated = true]; - optional int64 deviceProfileIdMSB = 6 [deprecated = true]; - optional int64 deviceProfileIdLSB = 7 [deprecated = true]; - string name = 8 [deprecated = true]; - string type = 9 [deprecated = true]; - optional string label = 10 [deprecated = true]; - optional string additionalInfo = 11 [deprecated = true]; - optional string conflictName = 12 [deprecated = true]; - optional int64 firmwareIdMSB = 13 [deprecated = true]; - optional int64 firmwareIdLSB = 14 [deprecated = true]; - optional bytes deviceDataBytes = 15 [deprecated = true]; - optional int64 softwareIdMSB = 16 [deprecated = true]; - optional int64 softwareIdLSB = 17 [deprecated = true]; string entity = 18; } @@ -233,24 +168,6 @@ message DeviceProfileUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - string name = 4 [deprecated = true]; - optional string description = 5 [deprecated = true]; - bool default = 6 [deprecated = true]; - string type = 7 [deprecated = true]; - optional string transportType = 8 [deprecated = true]; - optional string provisionType = 9 [deprecated = true]; - int64 defaultRuleChainIdMSB = 10 [deprecated = true]; - int64 defaultRuleChainIdLSB = 11 [deprecated = true]; - string defaultQueueName = 12 [deprecated = true]; - bytes profileDataBytes = 13 [deprecated = true]; - optional string provisionDeviceKey = 14 [deprecated = true]; - optional bytes image = 15 [deprecated = true]; - optional int64 firmwareIdMSB = 16 [deprecated = true]; - optional int64 firmwareIdLSB = 17 [deprecated = true]; - optional int64 softwareIdMSB = 18 [deprecated = true]; - optional int64 softwareIdLSB = 19 [deprecated = true]; - optional int64 defaultDashboardIdMSB = 20 [deprecated = true]; - optional int64 defaultDashboardIdLSB = 21 [deprecated = true]; string entity = 22; } @@ -258,24 +175,10 @@ message AssetProfileUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - string name = 4 [deprecated = true]; - optional string description = 5 [deprecated = true]; - bool default = 6 [deprecated = true]; - int64 defaultRuleChainIdMSB = 7 [deprecated = true]; - int64 defaultRuleChainIdLSB = 8 [deprecated = true]; - int64 defaultDashboardIdMSB = 9 [deprecated = true]; - int64 defaultDashboardIdLSB = 10 [deprecated = true]; - optional string defaultQueueName = 11 [deprecated = true]; - optional bytes image = 12 [deprecated = true]; string entity = 13; } message DeviceCredentialsUpdateMsg { - int64 deviceIdMSB = 1 [deprecated = true]; - int64 deviceIdLSB = 2 [deprecated = true]; - string credentialsType = 3 [deprecated = true]; - string credentialsId = 4 [deprecated = true]; - optional string credentialsValue = 5 [deprecated = true]; string entity = 6; } @@ -283,14 +186,6 @@ message AssetUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - optional int64 customerIdMSB = 4 [deprecated = true]; - optional int64 customerIdLSB = 5 [deprecated = true]; - string name = 6 [deprecated = true]; - string type = 7 [deprecated = true]; - optional string label = 8 [deprecated = true]; - optional string additionalInfo = 9 [deprecated = true]; - optional int64 assetProfileIdMSB = 10 [deprecated = true]; - optional int64 assetProfileIdLSB = 11 [deprecated = true]; string entity = 12; } @@ -298,14 +193,6 @@ message EntityViewUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - optional int64 customerIdMSB = 4 [deprecated = true]; - optional int64 customerIdLSB = 5 [deprecated = true]; - string name = 6 [deprecated = true]; - string type = 7 [deprecated = true]; - int64 entityIdMSB = 8 [deprecated = true]; - int64 entityIdLSB = 9 [deprecated = true]; - EdgeEntityType entityType = 10 [deprecated = true]; - optional string additionalInfo = 11 [deprecated = true]; string entity = 12; } @@ -313,20 +200,6 @@ message AlarmUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - string name = 4 [deprecated = true]; - string type = 5 [deprecated = true]; - string originatorType = 6 [deprecated = true]; - string originatorName = 7 [deprecated = true]; - string severity = 8 [deprecated = true]; - string status = 9 [deprecated = true]; - int64 startTs = 10 [deprecated = true]; - int64 endTs = 11 [deprecated = true]; - int64 ackTs = 12 [deprecated = true]; - int64 clearTs = 13 [deprecated = true]; - string details = 14 [deprecated = true]; - bool propagate = 15 [deprecated = true]; - bool propagateToOwner = 16 [deprecated = true]; - bool propagateToTenant = 17 [deprecated = true]; string entity = 18; } @@ -339,30 +212,11 @@ message CustomerUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - string title = 4 [deprecated = true]; - optional string country = 5 [deprecated = true]; - optional string state = 6 [deprecated = true]; - optional string city = 7 [deprecated = true]; - optional string address = 8 [deprecated = true]; - optional string address2 = 9 [deprecated = true]; - optional string zip = 10 [deprecated = true]; - optional string phone = 11 [deprecated = true]; - optional string email = 12 [deprecated = true]; - optional string additionalInfo = 13 [deprecated = true]; string entity = 14; } message RelationUpdateMsg { UpdateMsgType msgType = 1; - int64 fromIdMSB = 2 [deprecated = true]; - int64 fromIdLSB = 3 [deprecated = true]; - string fromEntityType = 4 [deprecated = true]; - int64 toIdMSB = 5 [deprecated = true]; - int64 toIdLSB = 6 [deprecated = true]; - string toEntityType = 7 [deprecated = true]; - string type = 8 [deprecated = true]; - optional string typeGroup = 9 [deprecated = true]; - string additionalInfo = 10 [deprecated = true]; string entity = 11; } @@ -370,13 +224,6 @@ message UserUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - optional int64 customerIdMSB = 4 [deprecated = true]; - optional int64 customerIdLSB = 5 [deprecated = true]; - string email = 6 [deprecated = true]; - string authority = 7 [deprecated = true]; - optional string firstName = 8 [deprecated = true]; - optional string lastName = 9 [deprecated = true]; - optional string additionalInfo = 10 [deprecated = true]; string entity = 11; } @@ -384,13 +231,7 @@ message WidgetsBundleUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - string title = 4 [deprecated = true]; - string alias = 5 [deprecated = true]; - optional bytes image = 6 [deprecated = true]; - bool isSystem = 7 [deprecated = true]; - optional string description = 8 [deprecated = true]; optional string widgets = 9; - optional int32 order = 10 [deprecated = true]; string entity = 11; } @@ -398,63 +239,24 @@ message WidgetTypeUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - optional string bundleAlias = 4 [deprecated = true]; - optional string alias = 5 [deprecated = true]; - optional string name = 6 [deprecated = true]; - optional string descriptorJson = 7 [deprecated = true]; - bool isSystem = 8 [deprecated = true]; - optional string image = 9 [deprecated = true]; - optional string description = 10 [deprecated = true]; - optional string fqn = 11 [deprecated = true]; - bool deprecated = 12 [deprecated = true]; - repeated string tags = 13 [deprecated = true]; string entity = 14; } message AdminSettingsUpdateMsg { - bool isSystem = 1 [deprecated = true]; - string key = 2 [deprecated = true]; - string jsonValue = 3 [deprecated = true]; string entity = 4; } message UserCredentialsUpdateMsg { - int64 userIdMSB = 1 [deprecated = true]; - int64 userIdLSB = 2 [deprecated = true]; - bool enabled = 3 [deprecated = true]; - string password = 4 [deprecated = true]; string entity = 5; } message TenantUpdateMsg { UpdateMsgType msgType = 1; - int64 idMSB = 2 [deprecated = true]; - int64 idLSB = 3 [deprecated = true]; - string title = 4 [deprecated = true]; - int64 profileIdMSB = 5 [deprecated = true]; - int64 profileIdLSB = 6 [deprecated = true]; - string region = 7 [deprecated = true]; - optional string country = 8 [deprecated = true]; - optional string state = 9 [deprecated = true]; - optional string city = 10 [deprecated = true]; - optional string address = 11 [deprecated = true]; - optional string address2 = 12 [deprecated = true]; - optional string zip = 13 [deprecated = true]; - optional string phone = 14 [deprecated = true]; - optional string email = 15 [deprecated = true]; - optional string additionalInfo = 16 [deprecated = true]; string entity = 17; } message TenantProfileUpdateMsg { UpdateMsgType msgType = 1; - int64 idMSB = 2 [deprecated = true]; - int64 idLSB = 3 [deprecated = true]; - string name = 4 [deprecated = true]; - optional string description = 5 [deprecated = true]; - bool default = 6 [deprecated = true]; - bool isolatedRuleChain = 7 [deprecated = true]; - bytes profileDataBytes = 8 [deprecated = true]; string entity = 9; } @@ -462,13 +264,6 @@ message ResourceUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - string title = 4 [deprecated = true]; - string resourceType = 5 [deprecated = true]; - string resourceKey = 6 [deprecated = true]; - string fileName = 7 [deprecated = true]; - optional string data = 8 [deprecated = true]; - optional string etag = 9 [deprecated = true]; - bool isSystem = 10 [deprecated = true]; string entity = 11; } @@ -535,12 +330,6 @@ message DeviceCredentialsRequestMsg { int64 deviceIdLSB = 2; } -message DeviceProfileDevicesRequestMsg { - option deprecated = true; - int64 deviceProfileIdMSB = 1; - int64 deviceProfileIdLSB = 2; -} - message WidgetBundleTypesRequestMsg { int64 widgetBundleIdMSB = 1; int64 widgetBundleIdLSB = 2; @@ -579,29 +368,10 @@ message RpcResponseMsg { string error = 2; } -enum EdgeEntityType { - DEVICE = 0; - ASSET = 1; -} - message OtaPackageUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - int64 deviceProfileIdMSB = 4 [deprecated = true]; - int64 deviceProfileIdLSB = 5 [deprecated = true]; - string type = 6 [deprecated = true]; - string title = 7 [deprecated = true]; - string version = 8 [deprecated = true]; - string tag = 9 [deprecated = true]; - optional string url = 10 [deprecated = true]; - optional string fileName = 11 [deprecated = true]; - optional string contentType = 12 [deprecated = true]; - optional string checksumAlgorithm = 13 [deprecated = true]; - optional string checksum = 14 [deprecated = true]; - optional int64 dataSize = 15 [deprecated = true]; - optional bytes data = 16 [deprecated = true]; - optional string additionalInfo = 17 [deprecated = true]; string entity = 18; } @@ -609,34 +379,9 @@ message QueueUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; int64 idLSB = 3; - int64 tenantIdMSB = 4 [deprecated = true]; - int64 tenantIdLSB = 5 [deprecated = true]; - string name = 6 [deprecated = true]; - string topic = 7 [deprecated = true]; - int32 pollInterval = 8 [deprecated = true]; - int32 partitions = 9 [deprecated = true]; - bool consumerPerPartition = 10 [deprecated = true]; - int64 packProcessingTimeout = 11 [deprecated = true]; - SubmitStrategyProto submitStrategy = 12 [deprecated = true]; - ProcessingStrategyProto processingStrategy = 13 [deprecated = true]; string entity = 14; } -message SubmitStrategyProto { - option deprecated = true; - string type = 1; - int32 batchSize = 2; -} - -message ProcessingStrategyProto { - option deprecated = true; - string type = 1; - int32 retries = 2; - double failurePercentage = 3; - int64 pauseBetweenRetries = 4; - int64 maxPauseBetweenRetries = 5; -} - /** * Main Messages; */ @@ -654,7 +399,6 @@ message UplinkMsg { repeated UserCredentialsRequestMsg userCredentialsRequestMsg = 10; repeated DeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 11; repeated DeviceRpcCallMsg deviceRpcCallMsg = 12; - repeated DeviceProfileDevicesRequestMsg deviceProfileDevicesRequestMsg = 13 [deprecated = true]; repeated WidgetBundleTypesRequestMsg widgetBundleTypesRequestMsg = 14; repeated EntityViewsRequestMsg entityViewsRequestMsg = 15; repeated AssetUpdateMsg assetUpdateMsg = 16; From 8df4c326cb643825b43f66ca17c260973921a095 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 19 Dec 2024 10:41:09 +0200 Subject: [PATCH 055/416] Fix Oauth2EdgeTest --- .../server/service/edge/EdgeContextComponent.java | 1 + .../service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index ce0725a627..d001cbc38b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -118,6 +118,7 @@ public class EdgeContextComponent { map.put(EdgeEventType.NOTIFICATION_RULE, notificationRuleProcessor); map.put(EdgeEventType.NOTIFICATION_TARGET, notificationTargetProcessor); map.put(EdgeEventType.NOTIFICATION_TEMPLATE, notificationTemplateProcessor); + map.put(EdgeEventType.OAUTH2_CLIENT, oAuth2ClientProcessor); map.put(EdgeEventType.OTA_PACKAGE, otaPackageProcessor); map.put(EdgeEventType.QUEUE, queueProcessor); map.put(EdgeEventType.RELATION, relationProcessor); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java index 6cfe5a6819..32dc3bca4c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.edge.rpc.processor.alarm; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; @@ -29,7 +28,6 @@ import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; @@ -45,9 +43,6 @@ import java.util.UUID; @TbCoreComponent public class AlarmEdgeProcessor extends BaseAlarmProcessor implements AlarmProcessor { - @Autowired - private EntityService entityService; - @Override public ListenableFuture processAlarmMsgFromEdge(TenantId tenantId, EdgeId edgeId, AlarmUpdateMsg alarmUpdateMsg) { log.trace("[{}] processAlarmMsgFromEdge [{}]", tenantId, alarmUpdateMsg); From 5c5dc474cbc2ba899e9aa61106a211981e122dec Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 19 Dec 2024 17:12:59 +0200 Subject: [PATCH 056/416] implemented partitioning --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 115 +++++++++++++++--- .../queue/DefaultTbCoreConsumerService.java | 16 ++- common/proto/src/main/proto/queue.proto | 12 ++ 4 files changed, 125 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 302e4b2511..966b5d7277 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,7 +29,9 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); - void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); + + void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7e5ea5b5ec..f2a09d0faf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -98,6 +98,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -184,7 +185,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); } catch (Exception e) { - log.warn("Failed to resolve partition for CalculatedField [{}], tenant [{}]. Reason: {}", + log.warn("Failed to resolve partition for CalculatedField [{}], tenant id [{}]. Reason: {}", cf.getId(), cf.getTenantId(), e.getMessage()); continue; } @@ -233,8 +234,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - calculatedFields.remove(entityId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(entityId.getId())); + cleanupEntity(entityId); + } + + private void cleanupEntity(CalculatedFieldId calculatedFieldId) { + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); } @Override @@ -245,7 +250,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); + onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); @@ -364,7 +369,34 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + String state = proto.getState(); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = state.isEmpty() ? JacksonUtil.fromString(state, CalculatedFieldEntityCtx.class) : null; + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + if (calculatedFieldEntityCtx != null) { + states.put(ctxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), state); + } else { + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } + } else { + log.debug("[{}] Calculated Field belongs to external partition {}", calculatedFieldId, tpi.getFullTopicName()); + } + } catch (Exception e) { + log.trace("Failed to process calculated field update state msg: [{}]", proto, e); + } + } + + @Override + public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); @@ -377,9 +409,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + } }); initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); @@ -398,12 +435,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); profileEntities.get(profileId).remove(entityId); - List statesToRemove = states.keySet().stream() - .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); - rocksDBService.deleteAll(statesToRemove); + List calculatedFieldIds = Stream.concat( + calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId), + calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + ).toList(); + calculatedFieldIds.forEach(cfId -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + } + }); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); @@ -414,11 +461,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState calculatedFieldState) { + TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + if (calculatedFieldState != null) { + msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); + } + clusterService.pushMsgToCore(tenantId, entityId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); + onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); } else { calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); @@ -428,8 +490,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return shouldReinit; } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { try { + cleanupEntity(calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + Set calculatedFieldIds = partitionedEntities.get(tpi); + if (calculatedFieldIds != null) { + calculatedFieldIds.remove(calculatedFieldId); + } calculatedFields.remove(calculatedFieldId); calculatedFieldsCtx.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); @@ -606,7 +674,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); + CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -616,8 +685,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (state.updateState(argumentValues)) { calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + TenantId tenantId = calculatedFieldCtx.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldCtx.getCfId()); + if (tpi.isMyPartition()) { + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + } boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); if (allArgsPresent) { @@ -626,7 +701,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + pushMsgToRuleEngine(tenantId, entityId, result); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 6cf42cb893..a686184b8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -322,6 +322,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChanged(profileUpdateMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { @@ -708,6 +710,18 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldStateMsg(calculatedFieldStateMsgProto, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field state message for entityId [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 76da257b32..153394525d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -807,6 +807,17 @@ message ProfileEntityMsgProto { bool deleted = 10; } +message CalculatedFieldStateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + string entityType = 5; + int64 entityIdMSB = 6; + int64 entityIdLSB = 7; + string state = 8; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1541,6 +1552,7 @@ message ToCoreMsg { CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; ProfileEntityMsgProto profileEntityMsg = 55; + CalculatedFieldStateMsgProto calculatedFieldStateMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ From 8abaf9e0e8a131bd66701f79fe24aad4a99cf88a Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 20 Dec 2024 15:28:21 +0200 Subject: [PATCH 057/416] Refactor EdgeContextComponent map --- .../service/edge/EdgeContextComponent.java | 52 ++++++------------- .../edge/rpc/processor/BaseEdgeProcessor.java | 2 + .../edge/rpc/processor/EdgeProcessor.java | 5 ++ .../processor/alarm/AlarmEdgeProcessor.java | 5 ++ .../comment/AlarmCommentEdgeProcessor.java | 5 ++ .../processor/asset/AssetEdgeProcessor.java | 5 ++ .../profile/AssetProfileEdgeProcessor.java | 5 ++ .../customer/CustomerEdgeProcessor.java | 5 ++ .../dashboard/DashboardEdgeProcessor.java | 6 +++ .../processor/device/DeviceEdgeProcessor.java | 5 ++ .../profile/DeviceProfileEdgeProcessor.java | 5 ++ .../processor/edge/EdgeEntityProcessor.java | 5 ++ .../entityview/EntityViewEdgeProcessor.java | 5 ++ .../NotificationRuleEdgeProcessor.java | 6 +++ .../NotificationTargetEdgeProcessor.java | 6 +++ .../NotificationTemplateEdgeProcessor.java | 6 +++ .../processor/oauth2/DomainEdgeProcessor.java | 6 +++ .../oauth2/OAuth2ClientEdgeProcessor.java | 6 +++ .../ota/OtaPackageEdgeProcessor.java | 6 +++ .../processor/queue/QueueEdgeProcessor.java | 6 +++ .../relation/RelationEdgeProcessor.java | 5 ++ .../resource/ResourceEdgeProcessor.java | 5 ++ .../rule/RuleChainEdgeProcessor.java | 6 +++ .../rule/RuleChainMetadataEdgeProcessor.java | 6 +++ .../settings/AdminSettingsEdgeProcessor.java | 6 +++ .../processor/tenant/TenantEdgeProcessor.java | 6 +++ .../tenant/TenantProfileEdgeProcessor.java | 6 +++ .../rpc/processor/user/UserEdgeProcessor.java | 6 +++ .../widget/WidgetBundleEdgeProcessor.java | 6 +++ .../widget/WidgetTypeEdgeProcessor.java | 6 +++ 30 files changed, 174 insertions(+), 36 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index d001cbc38b..fd0dff95de 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; @@ -87,50 +88,29 @@ import org.thingsboard.server.service.edge.rpc.processor.widget.WidgetTypeEdgePr import org.thingsboard.server.service.edge.rpc.sync.EdgeRequestsService; import org.thingsboard.server.service.executors.GrpcCallbackExecutorService; -import javax.annotation.PostConstruct; -import java.util.Collections; -import java.util.HashMap; +import java.util.EnumMap; +import java.util.List; import java.util.Map; @Lazy @Data +@Slf4j @Component @TbCoreComponent public class EdgeContextComponent { - private Map processorMap; - - @PostConstruct - private void initProcessorMap() { - Map map = new HashMap<>(); - map.put(EdgeEventType.ADMIN_SETTINGS, adminSettingsProcessor); - map.put(EdgeEventType.ALARM, alarmProcessor); - map.put(EdgeEventType.ALARM_COMMENT, alarmCommentProcessor); - map.put(EdgeEventType.ASSET, assetProcessor); - map.put(EdgeEventType.ASSET_PROFILE, assetProfileProcessor); - map.put(EdgeEventType.CUSTOMER, customerProcessor); - map.put(EdgeEventType.DASHBOARD, dashboardProcessor); - map.put(EdgeEventType.DEVICE, deviceProcessor); - map.put(EdgeEventType.DEVICE_PROFILE, deviceProfileProcessor); - map.put(EdgeEventType.DOMAIN, domainProcessor); - map.put(EdgeEventType.EDGE, edgeEntityProcessor); - map.put(EdgeEventType.ENTITY_VIEW, entityViewProcessor); - map.put(EdgeEventType.NOTIFICATION_RULE, notificationRuleProcessor); - map.put(EdgeEventType.NOTIFICATION_TARGET, notificationTargetProcessor); - map.put(EdgeEventType.NOTIFICATION_TEMPLATE, notificationTemplateProcessor); - map.put(EdgeEventType.OAUTH2_CLIENT, oAuth2ClientProcessor); - map.put(EdgeEventType.OTA_PACKAGE, otaPackageProcessor); - map.put(EdgeEventType.QUEUE, queueProcessor); - map.put(EdgeEventType.RELATION, relationProcessor); - map.put(EdgeEventType.RULE_CHAIN, ruleChainProcessor); - map.put(EdgeEventType.RULE_CHAIN_METADATA, ruleChainMetadataProcessor); - map.put(EdgeEventType.TB_RESOURCE, resourceProcessor); - map.put(EdgeEventType.TENANT, tenantProcessor); - map.put(EdgeEventType.TENANT_PROFILE, tenantProfileProcessor); - map.put(EdgeEventType.USER, userProcessor); - map.put(EdgeEventType.WIDGETS_BUNDLE, widgetBundleProcessor); - map.put(EdgeEventType.WIDGET_TYPE, widgetTypeProcessor); - this.processorMap = Collections.unmodifiableMap(map); + private final Map processorMap = new EnumMap<>(EdgeEventType.class); + + @Autowired + public EdgeContextComponent(List processors) { + processors.forEach(processor -> { + EdgeEventType eventType = processor.getEdgeEventType(); + if (eventType != null) { + processorMap.put(eventType, processor); + } else { + log.warn("Processor {} does not specify a supported EdgeEventType", processor.getClass().getSimpleName()); + } + }); } // services diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index c8647fb40a..f88fd4f156 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EdgeUtils; @@ -75,6 +76,7 @@ public abstract class BaseEdgeProcessor implements EdgeProcessor { protected static final Lock deviceCreationLock = new ReentrantLock(); protected static final Lock assetCreationLock = new ReentrantLock(); + @Lazy @Autowired protected EdgeContextComponent edgeCtx; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeProcessor.java index b4d4550966..98342998a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeProcessor.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.edge.rpc.processor; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.transport.TransportProtos; @@ -29,4 +30,8 @@ public interface EdgeProcessor { return null; } + default EdgeEventType getEdgeEventType() { + return null; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java index 32dc3bca4c..72ff03128a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java @@ -115,4 +115,9 @@ public class AlarmEdgeProcessor extends BaseAlarmProcessor implements AlarmProce return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.ALARM; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentEdgeProcessor.java index ef13144ba8..90c2e8031e 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/comment/AlarmCommentEdgeProcessor.java @@ -90,4 +90,9 @@ public class AlarmCommentEdgeProcessor extends BaseAlarmProcessor implements Ala return Futures.transform(Futures.allAsList(delFutures), voids -> null, dbCallbackExecutorService); } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.ALARM_COMMENT; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java index 81145a6195..51815bc3aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java @@ -141,4 +141,9 @@ public class AssetEdgeProcessor extends BaseAssetProcessor implements AssetProce asset.setCustomerId(customerUUID); } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.ASSET; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java index df3d74ceaa..53f35d3aac 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java @@ -136,4 +136,9 @@ public class AssetProfileEdgeProcessor extends BaseAssetProfileProcessor impleme assetProfile.setDefaultDashboardId(defaultDashboardUUID != null ? new DashboardId(defaultDashboardUUID) : null); } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.ASSET_PROFILE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java index 2ecec3893e..fc70e8964b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java @@ -95,4 +95,9 @@ public class CustomerEdgeProcessor extends BaseEdgeProcessor { } } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.CUSTOMER; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java index e4842a16db..8b89eb51c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.ShortCustomerInfo; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; @@ -130,4 +131,9 @@ public class DashboardEdgeProcessor extends BaseDashboardProcessor implements Da return assignedCustomers; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.DASHBOARD; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java index daf687fe6f..db407b23b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java @@ -274,4 +274,9 @@ public class DeviceEdgeProcessor extends BaseDeviceProcessor implements DevicePr device.setCustomerId(customerUUID); } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.DEVICE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java index 31a007219d..eef994be93 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java @@ -136,4 +136,9 @@ public class DeviceProfileEdgeProcessor extends BaseDeviceProfileProcessor imple deviceProfile.setDefaultDashboardId(defaultDashboardUUID != null ? new DashboardId(defaultDashboardUUID) : null); } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.DEVICE_PROFILE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java index 7c1f907cb0..7e969d5db4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java @@ -113,4 +113,9 @@ public class EdgeEntityProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.EDGE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java index a46ac3acdd..620e9c9c34 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java @@ -137,4 +137,9 @@ public class EntityViewEdgeProcessor extends BaseEntityViewProcessor implements entityView.setCustomerId(customerUUID); } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.ENTITY_VIEW; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationRuleEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationRuleEdgeProcessor.java index d5f68f5427..0322c17b74 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationRuleEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationRuleEdgeProcessor.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.NotificationRuleId; import org.thingsboard.server.common.data.notification.rule.NotificationRule; import org.thingsboard.server.dao.notification.NotificationRuleService; @@ -65,4 +66,9 @@ public class NotificationRuleEdgeProcessor extends BaseEdgeProcessor { return downlinkMsg; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.NOTIFICATION_RULE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTargetEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTargetEdgeProcessor.java index 08d0efcaad..a9a6044812 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTargetEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTargetEdgeProcessor.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.NotificationTargetId; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; import org.thingsboard.server.dao.notification.NotificationTargetService; @@ -65,4 +66,9 @@ public class NotificationTargetEdgeProcessor extends BaseEdgeProcessor { return downlinkMsg; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.NOTIFICATION_TARGET; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTemplateEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTemplateEdgeProcessor.java index 1a4227a5fc..589db4912b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTemplateEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationTemplateEdgeProcessor.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.NotificationTemplateId; import org.thingsboard.server.common.data.notification.template.NotificationTemplate; import org.thingsboard.server.dao.notification.NotificationTemplateService; @@ -65,4 +66,9 @@ public class NotificationTemplateEdgeProcessor extends BaseEdgeProcessor { return downlinkMsg; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.NOTIFICATION_TEMPLATE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/DomainEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/DomainEdgeProcessor.java index a6016c97d9..4eb3e4001d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/DomainEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/DomainEdgeProcessor.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.domain.DomainInfo; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.DomainId; import org.thingsboard.server.common.data.oauth2.OAuth2Client; import org.thingsboard.server.dao.domain.DomainService; @@ -71,4 +72,9 @@ public class DomainEdgeProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.DOMAIN; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2ClientEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2ClientEdgeProcessor.java index cc1c361824..512f03f666 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2ClientEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2ClientEdgeProcessor.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.OAuth2ClientId; import org.thingsboard.server.common.data.oauth2.OAuth2Client; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -64,4 +65,9 @@ public class OAuth2ClientEdgeProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.OAUTH2_CLIENT; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java index c09d735f5b..0cc90a3ef2 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; @@ -59,4 +60,9 @@ public class OtaPackageEdgeProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.OTA_PACKAGE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java index 937049f447..8da1a18f86 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -59,4 +60,9 @@ public class QueueEdgeProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.QUEUE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java index bc9b955033..9fc234c114 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java @@ -95,4 +95,9 @@ public class RelationEdgeProcessor extends BaseRelationProcessor implements Rela .build(); } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.RELATION; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java index 1d5ebc188f..1b1829743b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java @@ -98,4 +98,9 @@ public class ResourceEdgeProcessor extends BaseResourceProcessor implements Reso return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.TB_RESOURCE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java index 72d6565b92..80b0963d0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -79,4 +80,9 @@ public class RuleChainEdgeProcessor extends BaseEdgeProcessor { return downlinkMsg; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.RULE_CHAIN; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainMetadataEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainMetadataEdgeProcessor.java index eedfc835ae..f85045093c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainMetadataEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainMetadataEdgeProcessor.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -51,4 +52,9 @@ public class RuleChainMetadataEdgeProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.RULE_CHAIN_METADATA; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java index bdfa597ae7..2dc6f0e3f8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java @@ -21,6 +21,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.gen.edge.v1.AdminSettingsUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -44,4 +45,9 @@ public class AdminSettingsEdgeProcessor extends BaseEdgeProcessor { .build(); } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.ADMIN_SETTINGS; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java index 311975501d..4234e0ca32 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; @@ -56,4 +57,9 @@ public class TenantEdgeProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.TENANT; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java index b02a8e2ce3..0312389e0d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; @@ -51,4 +52,9 @@ public class TenantProfileEdgeProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.TENANT_PROFILE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java index 1d2f1f364a..f7ab19af60 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -72,4 +73,9 @@ public class UserEdgeProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.USER; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java index 5ea5817639..24217d80e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -62,4 +63,9 @@ public class WidgetBundleEdgeProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.WIDGETS_BUNDLE; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java index f3ea3ec1ef..c626a9ad0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -59,4 +60,9 @@ public class WidgetTypeEdgeProcessor extends BaseEdgeProcessor { return null; } + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.WIDGET_TYPE; + } + } From eaad06309682c8280439b2f8814f33d63442f780 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 20 Dec 2024 15:33:28 +0200 Subject: [PATCH 058/416] Remove log.warn from EdgeContextComponent --- .../thingsboard/server/service/edge/EdgeContextComponent.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index fd0dff95de..9faafa6379 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -107,8 +107,6 @@ public class EdgeContextComponent { EdgeEventType eventType = processor.getEdgeEventType(); if (eventType != null) { processorMap.put(eventType, processor); - } else { - log.warn("Processor {} does not specify a supported EdgeEventType", processor.getClass().getSimpleName()); } }); } From 2a41c8b45123e43fc34def1a954e098298c7339e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 20 Dec 2024 16:16:58 +0200 Subject: [PATCH 059/416] implemented logic to fetch telemetry if states were not fetched --- ...efaultCalculatedFieldExecutionService.java | 171 ++++++++++-------- .../ctx/state/SingleValueArgumentEntry.java | 10 + .../cf/ctx/state/TsRollingArgumentEntry.java | 13 +- 3 files changed, 118 insertions(+), 76 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f2a09d0faf..1128a75008 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -81,12 +81,13 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -97,6 +98,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -200,11 +202,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas var future = calculatedFieldExecutor.submit(() -> { try { for (CalculatedField cf : partition) { - if (EntityType.ASSET_PROFILE.equals(cf.getEntityId().getEntityType()) || EntityType.DEVICE_PROFILE.equals(cf.getEntityId().getEntityType())) { - getOrFetchFromDBProfileEntities(cf.getTenantId(), cf.getEntityId()) + EntityId cfEntityId = cf.getEntityId(); + if (isProfileEntity(cfEntityId)) { + getOrFetchFromDBProfileEntities(cf.getTenantId(), cfEntityId) .forEach(entityId -> restoreState(cf, entityId)); } else { - restoreState(cf, cf.getEntityId()); + restoreState(cf, cfEntityId); } } } catch (Throwable t) { @@ -272,9 +275,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { + Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { - initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArguments, callback); + initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); }); }); } @@ -473,7 +479,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldState != null) { msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); } - clusterService.pushMsgToCore(tenantId, entityId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { @@ -556,35 +562,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } - private void fetchCommonArguments(CalculatedFieldCtx calculatedFieldCtx, TbCallback callback, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - - calculatedFieldCtx.getArguments().forEach((key, argument) -> { - if (!EntityType.DEVICE_PROFILE.equals(argument.getEntityId().getEntityType()) && - !EntityType.ASSET_PROFILE.equals(argument.getEntityId().getEntityType())) { - futures.add(Futures.transform(fetchKvEntry(calculatedFieldCtx.getTenantId(), argument.getEntityId(), argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - } - }); - - Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { - @Override - public void onSuccess(List results) { - onComplete.accept(argumentValues); - } - - @Override - public void onFailure(Throwable t) { - log.error("Failed to fetch common arguments", t); - callback.onFailure(t); - } - }, calculatedFieldCallbackExecutor); - } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); } @@ -595,7 +572,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCtx.getArguments().forEach((key, argument) -> { if (!commonArguments.containsKey(key)) { - futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx, entityId, argument), + futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx.getTenantId(), entityId, argument), result -> { argumentValues.put(key, result); return result; @@ -618,10 +595,25 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { - TenantId tenantId = calculatedFieldCtx.getTenantId(); + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { + Map argumentValues = new HashMap<>(); + List> futures = new ArrayList<>(); + necessaryArguments.forEach((key, argument) -> { + futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + }); + return Futures.transform(Futures.allAsList(futures), results -> { + onComplete.accept(argumentValues); + return null; + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) + EntityId entityId = isProfileEntity(argumentEntityId) ? targetEntityId : argumentEntityId; return fetchKvEntry(tenantId, entityId, argument); @@ -654,11 +646,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? ArgumentEntry.createTsRollingArgument(Collections.emptyList()) : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldCallbackExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createSingleValueArgument(kvEntry.get()); + } else { + return SingleValueArgumentEntry.EMPTY; + } + }, calculatedFieldCallbackExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -676,48 +674,67 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Predicate> allArgsPresent = (args) -> + args.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !args.containsValue(SingleValueArgumentEntry.EMPTY) && !args.containsValue(TsRollingArgumentEntry.EMPTY); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + TenantId tenantId = calculatedFieldCtx.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + } + + if (allArgsPresent.test(state.getArguments())) { + performCalculation(calculatedFieldCtx, state, entityId); + } + } + }; CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + if (!allKeysPresent) { - if (state == null) { - state = createStateByType(calculatedFieldCtx.getCfType()); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + return; } - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - TenantId tenantId = calculatedFieldCtx.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldCtx.getCfId()); - if (tpi.isMyPartition()) { - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); - } + performUpdateState.accept(state); + } - boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); - if (allArgsPresent) { - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(tenantId, entityId, result); - } - } + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId) { + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + } + } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); - } - }, MoreExecutors.directExecutor()); + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); } - } + }, MoreExecutors.directExecutor()); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, null); + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); } return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } @@ -725,8 +742,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTE".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTE".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); @@ -749,4 +766,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } + private boolean isProfileEntity(EntityId entityId) { + return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 81a57580db..3f4fd5bdce 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); + private long ts; private Object value; @@ -39,6 +41,14 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.value = entry.getValue(); } + /** + * Internal constructor to create immutable SingleValueArgumentEntry.EMPTY + * */ + private SingleValueArgumentEntry(int ignored) { + this.ts = System.currentTimeMillis(); + this.value = null; + } + @Override public ArgumentType getType() { return ArgumentType.SINGLE_VALUE; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 49aae15ac1..4ffa391550 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -29,6 +29,8 @@ import java.util.TreeMap; @Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { + public static final ArgumentEntry EMPTY = new TsRollingArgumentEntry(0); + private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; private TreeMap tsRecords = new TreeMap<>(); @@ -37,6 +39,13 @@ public class TsRollingArgumentEntry implements ArgumentEntry { addAllTsRecords(tsRecords); } + /** + * Internal constructor to create immutable TsRollingArgumentEntry.EMPTY + */ + private TsRollingArgumentEntry(int ignored) { + this.tsRecords = new TreeMap<>(); + } + @Override public ArgumentType getType() { return ArgumentType.TS_ROLLING; @@ -50,7 +59,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { @Override public boolean hasUpdatedValue(ArgumentEntry entry) { - return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); + return entry instanceof SingleValueArgumentEntry ? + !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()) : + !tsRecords.keySet().containsAll(((TsRollingArgumentEntry) entry).getTsRecords().keySet()); } @Override From 2d65a6c457182e2f7412f083b1be7069c47d7f2d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 23 Dec 2024 12:20:58 +0200 Subject: [PATCH 060/416] removed wildcard imports usage --- .../server/controller/BaseController.java | 62 ++++++++++++++++++- .../entitiy/EntityStateSourcingListener.java | 9 ++- .../DefaultTelemetrySubscriptionService.java | 17 +++-- .../rule/engine/util/TenantIdLoader.java | 33 +++++++++- .../rule/engine/util/TenantIdLoaderTest.java | 27 +++++++- 5 files changed, 136 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 2d990d708d..9a980d009d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -42,7 +42,27 @@ import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HomeDashboardInfo; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantInfo; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.alarm.AlarmInfo; @@ -57,7 +77,37 @@ import org.thingsboard.server.common.data.edge.EdgeInfo; import org.thingsboard.server.common.data.exception.EntityVersionMismatchException; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AlarmCommentId; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.DomainId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.MobileAppBundleId; +import org.thingsboard.server.common.data.id.MobileAppId; +import org.thingsboard.server.common.data.id.NotificationTargetId; +import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.RpcId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TbResourceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.mobile.bundle.MobileAppBundle; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; @@ -140,7 +190,13 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 83ce48c24e..ab8246ccf5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -23,7 +23,14 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index bf66a9f868..2ddfd22db3 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -29,7 +29,11 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; +import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -49,7 +53,13 @@ import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; -import java.util.*; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -231,8 +241,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .onlyLatest(true) .callback(new FutureCallback<>() { @Override - public void onSuccess(@Nullable Void tmp) { - } + public void onSuccess(@Nullable Void tmp) {} @Override public void onFailure(Throwable t) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java index 4311d912e4..445205e88e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java @@ -19,7 +19,38 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.DomainId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.MobileAppBundleId; +import org.thingsboard.server.common.data.id.MobileAppId; +import org.thingsboard.server.common.data.id.NotificationRequestId; +import org.thingsboard.server.common.data.id.NotificationRuleId; +import org.thingsboard.server.common.data.id.NotificationTargetId; +import org.thingsboard.server.common.data.id.NotificationTemplateId; +import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.QueueStatsId; +import org.thingsboard.server.common.data.id.RpcId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TbResourceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.rule.RuleNode; import java.util.UUID; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index a879c0e3a2..921b9e26d7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -23,8 +23,23 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.AbstractListeningExecutor; -import org.thingsboard.rule.engine.api.*; -import org.thingsboard.server.common.data.*; +import org.thingsboard.rule.engine.api.RuleEngineAlarmService; +import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService; +import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache; +import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; +import org.thingsboard.rule.engine.api.RuleEngineRpcService; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; @@ -32,7 +47,13 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.NotificationId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.mobile.bundle.MobileAppBundle; import org.thingsboard.server.common.data.notification.NotificationRequest; From 1482d1c4bb9983f18ab9d6abb5405f1a68496ec9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 23 Dec 2024 17:00:10 +0200 Subject: [PATCH 061/416] added logic to avoid looping --- .../cf/CalculatedFieldExecutionService.java | 3 +- ...efaultCalculatedFieldExecutionService.java | 57 +++++++------------ .../DefaultTelemetrySubscriptionService.java | 4 +- .../thingsboard/server/common/msg/TbMsg.java | 30 ++++++++-- common/message/src/main/proto/tbmsg.proto | 7 +++ .../engine/api/AttributesSaveRequest.java | 10 +++- .../engine/api/TimeseriesSaveRequest.java | 10 +++- .../engine/telemetry/TbMsgTimeseriesNode.java | 1 + 8 files changed, 76 insertions(+), 46 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 966b5d7277..ee9b6d115c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; @@ -27,7 +28,7 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry); void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 1128a75008..e5c119bba6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -18,11 +18,7 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.*; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; @@ -41,25 +37,8 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.kv.*; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -301,7 +280,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { try { EntityType entityType = entityId.getEntityType(); if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { @@ -326,7 +305,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas )); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, calculatedFieldIds, updatedTelemetry); } }); } @@ -335,7 +314,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); @@ -347,12 +326,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); } } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds); } log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } @@ -583,7 +562,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, Collections.emptyList()); callback.onSuccess(); } @@ -671,7 +650,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); @@ -693,7 +672,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (allArgsPresent.test(state.getArguments())) { - performCalculation(calculatedFieldCtx, state, entityId); + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } } }; @@ -714,13 +693,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas performUpdateState.accept(state); } - private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId) { + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, calculatedFieldIds); } } @@ -739,13 +718,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } - private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { + private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { String type = calculatedFieldResult.getType(); TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); - TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); + if (calculatedFieldIds.contains(calculatedFieldId)) { + throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); + } + calculatedFieldIds.add(calculatedFieldId); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 2ddfd22db3..e35d3cead6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -152,7 +152,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getCalculatedFieldIds(), request.getEntries()); return saveFuture; } @@ -168,7 +168,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getCalculatedFieldIds(), request.getEntries()); } @Override diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 64e05770fe..993fa24ff3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -34,6 +34,8 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.UUID; @@ -64,6 +66,8 @@ public final class TbMsg implements Serializable { private final UUID correlationId; private final Integer partition; + private final List calculatedFieldIds; + @Getter(value = AccessLevel.NONE) @JsonIgnore //This field is not serialized because we use queues and there is no need to do it @@ -112,7 +116,7 @@ public final class TbMsg implements Serializable { } private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, TbMsgProcessingCtx ctx, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List calculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { this.id = id != null ? id : UUID.randomUUID(); this.queueName = queueName; if (ts > 0) { @@ -139,6 +143,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = ruleNodeId; this.correlationId = correlationId; this.partition = partition; + this.calculatedFieldIds = calculatedFieldIds; this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx(); this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } @@ -200,6 +205,7 @@ public final class TbMsg implements Serializable { RuleNodeId ruleNodeId = null; UUID correlationId = null; Integer partition = null; + List calculatedFieldIds = new ArrayList<>(); if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); } @@ -214,6 +220,14 @@ public final class TbMsg implements Serializable { partition = proto.getPartition(); } + for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(calculatedFieldId); + } + TbMsgProcessingCtx ctx; if (proto.hasCtx()) { ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); @@ -224,7 +238,7 @@ public final class TbMsg implements Serializable { TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId, - metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } @@ -343,10 +357,12 @@ public final class TbMsg implements Serializable { protected RuleNodeId ruleNodeId; protected UUID correlationId; protected Integer partition; + protected List calculatedFieldIds; protected TbMsgProcessingCtx ctx; protected TbMsgCallback callback; - TbMsgBuilder() {} + TbMsgBuilder() { + } TbMsgBuilder(TbMsg tbMsg) { this.queueName = tbMsg.queueName; @@ -363,6 +379,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = tbMsg.ruleNodeId; this.correlationId = tbMsg.correlationId; this.partition = tbMsg.partition; + this.calculatedFieldIds = tbMsg.calculatedFieldIds; this.ctx = tbMsg.ctx; this.callback = tbMsg.callback; } @@ -454,6 +471,11 @@ public final class TbMsg implements Serializable { return this; } + public TbMsgBuilder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public TbMsgBuilder ctx(TbMsgProcessingCtx ctx) { this.ctx = ctx; return this; @@ -465,7 +487,7 @@ public final class TbMsg implements Serializable { } public TbMsg build() { - return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); } public String toString() { diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index fc9265aa14..36ab09f8d4 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -70,4 +70,11 @@ message TbMsgProto { int64 correlationIdMSB = 20; int64 correlationIdLSB = 21; int32 partition = 22; + + repeated CalculatedFieldIdProto calculatedFields = 23; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index 22fa8de6de..9747a6033e 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -22,6 +22,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -40,6 +41,7 @@ public class AttributesSaveRequest { private final AttributeScope scope; private final List entries; private final boolean notifyDevice; + private final List calculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -53,6 +55,7 @@ public class AttributesSaveRequest { private AttributeScope scope; private List entries; private boolean notifyDevice = true; + private List calculatedFieldIds; private FutureCallback callback; Builder() {} @@ -100,6 +103,11 @@ public class AttributesSaveRequest { return this; } + public Builder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -120,7 +128,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, calculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 2b5881212d..12afa2d939 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -40,6 +41,7 @@ public class TimeseriesSaveRequest { private final long ttl; private final boolean saveLatest; private final boolean onlyLatest; + private final List calculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -56,6 +58,7 @@ public class TimeseriesSaveRequest { private FutureCallback callback; private boolean saveLatest = true; private boolean onlyLatest; + private List calculatedFieldIds; Builder() {} @@ -103,6 +106,11 @@ public class TimeseriesSaveRequest { return this; } + public Builder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -123,7 +131,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, calculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 27f45feb47..386e56320a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -112,6 +112,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entries(tsKvEntryList) .ttl(ttl) .saveLatest(!config.isSkipLatestPersistence()) + .calculatedFieldIds(msg.getCalculatedFieldIds()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } From a9f39e4917d0da31bdc89a45a8c5df344eec9d1f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 24 Dec 2024 12:50:27 +0200 Subject: [PATCH 062/416] added cache class --- .../service/cf/CalculatedFieldCache.java | 43 +++ .../cf/DefaultCalculatedFieldCache.java | 212 +++++++++++++ ...efaultCalculatedFieldExecutionService.java | 279 ++++++++++-------- .../thingsboard/server/common/msg/TbMsg.java | 15 +- common/proto/src/main/proto/queue.proto | 29 +- .../engine/telemetry/TbMsgAttributesNode.java | 1 + 6 files changed, 454 insertions(+), 125 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java new file mode 100644 index 0000000000..f953e57cc3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; +import java.util.Set; + +public interface CalculatedFieldCache { + + CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + + CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + + Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + + void evict(CalculatedFieldId calculatedFieldId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java new file mode 100644 index 0000000000..1fc2a90e07 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -0,0 +1,212 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import jakarta.annotation.PostConstruct; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldCache implements CalculatedFieldCache { + + private final Lock calculatedFieldFetchLock = new ReentrantLock(); + + private final CalculatedFieldService calculatedFieldService; + private final AssetService assetService; + private final DeviceService deviceService; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + + @PostConstruct + public void init() { + // to discuss: fetch on start or fetch on demand + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + } + + @Override + public CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + CalculatedField calculatedField = calculatedFields.get(calculatedFieldId); + if (calculatedField == null) { + calculatedFieldFetchLock.lock(); + try { + calculatedField = calculatedFields.get(calculatedFieldId); + if (calculatedField == null) { + calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + if (calculatedField != null) { + calculatedFields.put(calculatedFieldId, calculatedField); + log.debug("[{}] Fetch calculated field into cache: {}", calculatedFieldId, calculatedField); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field in cache: {}", calculatedFieldId, calculatedField); + return calculatedField; + } + + @Override + public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + List cfLinks = calculatedFieldLinks.get(calculatedFieldId); + if (cfLinks == null) { + calculatedFieldFetchLock.lock(); + try { + cfLinks = calculatedFieldLinks.get(calculatedFieldId); + if (cfLinks == null) { + cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cfLinks != null) { + calculatedFieldLinks.put(calculatedFieldId, cfLinks); + log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field links in cache: {}", calculatedFieldId, cfLinks); + return cfLinks; + } + + @Override + public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + List cfLinks = entityIdCalculatedFieldLinks.get(entityId); + if (cfLinks == null) { + calculatedFieldFetchLock.lock(); + try { + cfLinks = entityIdCalculatedFieldLinks.get(entityId); + if (cfLinks == null) { + cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + if (cfLinks != null) { + entityIdCalculatedFieldLinks.put(entityId, cfLinks); + log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field links by entity id in cache: {}", entityId, cfLinks); + return cfLinks; + } + + @Override + public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + calculatedFieldFetchLock.lock(); + try { + ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + CalculatedField calculatedField = getCalculatedField(tenantId, calculatedFieldId); + if (calculatedField != null) { + ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); + calculatedFieldsCtx.put(calculatedFieldId, ctx); + log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); + return ctx; + } + + @Override + public Set getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) { + Set entities = profileEntities.get(entityProfileId); + if (entities == null) { + calculatedFieldFetchLock.lock(); + try { + entities = profileEntities.get(entityProfileId); + if (entities == null) { + entities = switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> + throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); + return entities; + } + + @Override + public void evict(CalculatedFieldId calculatedFieldId) { + CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldLinks.remove(calculatedFieldId); + log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldsCtx.remove(calculatedFieldId); + log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); + log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index e5c119bba6..ff78d2925a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -18,7 +18,11 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.*; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; @@ -37,8 +41,23 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.*; -import org.thingsboard.server.common.data.kv.*; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -46,10 +65,8 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -67,8 +84,8 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -77,7 +94,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -90,10 +106,9 @@ import static org.thingsboard.server.common.data.DataConstants.SCOPE; public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { private final CalculatedFieldService calculatedFieldService; - private final AssetService assetService; - private final DeviceService deviceService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; + private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; @@ -103,13 +118,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); - private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private static final int MAX_LAST_RECORDS_VALUE = 1024; @Value("${calculatedField.initFetchPackSize:50000}") @@ -123,20 +133,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(this::fetchCalculatedFields); - } - - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } @PreDestroy public void stop() { + super.stop(); if (calculatedFieldExecutor != null) { calculatedFieldExecutor.shutdownNow(); } @@ -183,7 +184,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas for (CalculatedField cf : partition) { EntityId cfEntityId = cf.getEntityId(); if (isProfileEntity(cfEntityId)) { - getOrFetchFromDBProfileEntities(cf.getTenantId(), cfEntityId) + calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId) .forEach(entityId -> restoreState(cf, entityId)); } else { restoreState(cf, cfEntityId); @@ -206,7 +207,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (storedState != null) { CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); - calculatedFieldsCtx.putIfAbsent(cf.getId(), new CalculatedFieldCtx(cf, tbelInvokeService)); states.put(ctxId, restoredCtx); log.info("Restored state for CalculatedField [{}]", cf.getId()); } else { @@ -220,7 +220,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - calculatedFields.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); } @@ -235,7 +234,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -245,8 +244,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (cf != null) { EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); - calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -258,7 +256,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { + calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); }); }); @@ -290,8 +288,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); + List cfLinks = calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId); + Optional.ofNullable(profileId).ifPresent(id -> calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); @@ -316,8 +314,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); + CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); @@ -326,7 +324,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); + calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); } else { updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); } @@ -353,28 +351,61 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entry.getKey(); } + private Object deserializeObjectProto(TransportProtos.ObjectProto objectProto) { + try { + String type = objectProto.getType(); + String value = objectProto.getValue(); + return switch (type) { + case "java.lang.String" -> value; + case "java.lang.Integer" -> Integer.parseInt(value); + case "java.lang.Long" -> Long.parseLong(value); + case "java.lang.Double" -> Double.parseDouble(value); + case "java.lang.Boolean" -> Boolean.parseBoolean(value); + default -> throw new IllegalArgumentException("Unsupported object type: " + type); + }; + } catch (Exception e) { + log.error("Failed to deserialize ObjectProto: [{}]", objectProto, e); + return null; + } + } + @Override public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - String state = proto.getState(); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = state.isEmpty() ? JacksonUtil.fromString(state, CalculatedFieldEntityCtx.class) : null; + List calculatedFieldIds = new ArrayList<>(); + for (TransportProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { + CalculatedFieldId cfId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(cfId); + } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); - if (calculatedFieldEntityCtx != null) { - states.put(ctxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), state); + Map argumentsMap = new HashMap<>(); + proto.getArgumentsMap().forEach((key, entryProto) -> { + ArgumentEntry argumentEntry; + if (entryProto.hasTsRecords()) { + TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); + entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> { + Object value = deserializeObjectProto(objectProto); + tsRollingArgumentEntry.getTsRecords().put(ts, value); + }); + argumentEntry = tsRollingArgumentEntry; + } else if (entryProto.hasSingleValue()) { + TransportProtos.SingleValueProto singleRecordProto = entryProto.getSingleValue(); + Object value = deserializeObjectProto(singleRecordProto.getValue()); + argumentEntry = new SingleValueArgumentEntry(singleRecordProto.getTs(), value); } else { - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); } - } else { - log.debug("[{}] Calculated Field belongs to external partition {}", calculatedFieldId, tpi.getFullTopicName()); - } + argumentsMap.put(key, argumentEntry); + }); + + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); } @@ -389,8 +420,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.get(oldProfileId).remove(entityId); - profileEntities.computeIfAbsent(newProfileId, id -> new HashSet<>()).add(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { @@ -400,7 +431,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); } }); @@ -419,12 +450,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.get(profileId).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); List calculatedFieldIds = Stream.concat( - calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId), - calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId) + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); calculatedFieldIds.forEach(cfId -> { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); @@ -433,12 +462,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); } }); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); } } catch (Exception e) { @@ -446,7 +475,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState calculatedFieldState) { + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) @@ -455,20 +484,45 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .setEntityType(entityId.getEntityType().name()) .setEntityIdMSB(entityId.getId().getMostSignificantBits()) .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - if (calculatedFieldState != null) { - msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); + + if (argumentValues != null) { + argumentValues.forEach((key, argumentEntry) -> { + TransportProtos.ArgumentEntryProto.Builder argumentEntryProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); + + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); + + tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> { + TransportProtos.ObjectProto.Builder objectProtoBuilder = TransportProtos.ObjectProto.newBuilder() + .setType(value.getClass().getName()) + .setValue(value.toString()); + tsRollingProtoBuilder.putTsRecords(ts, objectProtoBuilder.build()); + }); + + argumentEntryProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + TransportProtos.SingleValueProto.Builder singleRecordProtoBuilder = TransportProtos.SingleValueProto.newBuilder() + .setTs(singleValueArgumentEntry.getTs()) + .setValue(TransportProtos.ObjectProto.newBuilder() + .setType(singleValueArgumentEntry.getValue().getClass().getName()) + .setValue(singleValueArgumentEntry.getValue().toString()) + .build()); + argumentEntryProtoBuilder.setSingleValue(singleRecordProtoBuilder.build()); + } + + msgBuilder.putArguments(key, argumentEntryProtoBuilder.build()); + }); } + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); } else { - calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); - calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -483,8 +537,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldIds != null) { calculatedFieldIds.remove(calculatedFieldId); } - calculatedFields.remove(calculatedFieldId); - calculatedFieldsCtx.remove(calculatedFieldId); + calculatedFieldCache.evict(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) @@ -497,28 +550,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculatedField getOrFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); - } - - private Set getOrFetchFromDBProfileEntities(TenantId tenantId, EntityId entityProfileId) { - return switch (entityProfileId.getEntityType()) { - case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set assetIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); - return assetIds; - }); - case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set deviceIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); - return deviceIds; - }); - default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); - }; - } - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -537,7 +568,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(tenantId, cfId, tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -562,7 +593,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, Collections.emptyList()); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); callback.onSuccess(); } @@ -651,46 +682,47 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { + TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); - - Predicate> allArgsPresent = (args) -> - args.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !args.containsValue(SingleValueArgumentEntry.EMPTY) && !args.containsValue(TsRollingArgumentEntry.EMPTY); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - TenantId tenantId = calculatedFieldCtx.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); states.put(entityCtxId, calculatedFieldEntityCtx); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + } } + }; - if (allArgsPresent.test(state.getArguments())) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); - } - } - }; + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - if (!allKeysPresent) { + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + if (!allKeysPresent) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - return; + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + return; + } + performUpdateState.accept(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } - performUpdateState.accept(state); } private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { @@ -724,6 +756,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); + if (calculatedFieldIds == null) { + calculatedFieldIds = new ArrayList<>(); + } if (calculatedFieldIds.contains(calculatedFieldId)) { throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 993fa24ff3..0805175f77 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -191,6 +192,16 @@ public final class TbMsg implements Serializable { builder.setPartition(msg.getPartition()); } + if (msg.getCalculatedFieldIds() != null) { + for (CalculatedFieldId calculatedFieldId : msg.getCalculatedFieldIds()) { + MsgProtos.CalculatedFieldIdProto calculatedFieldIdProto = MsgProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .build(); + builder.addCalculatedFields(calculatedFieldIdProto); + } + } + builder.setCtx(msg.ctx.toProto()); return builder.build().toByteArray(); } @@ -495,8 +506,8 @@ public final class TbMsg implements Serializable { ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + - ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", ctx=" + this.ctx + - ", callback=" + this.callback + ")"; + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", calculatedFields=" + this.calculatedFieldIds + + ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; } } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 3df19e313e..f6f7afd5d8 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -817,7 +817,34 @@ message CalculatedFieldStateMsgProto { string entityType = 5; int64 entityIdMSB = 6; int64 entityIdLSB = 7; - string state = 8; + repeated CalculatedFieldIdProto calculatedFields = 8; + map arguments = 9; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; +} + +message ArgumentEntryProto { + oneof entry_type { + TsRollingProto tsRecords = 1; + SingleValueProto singleValue = 2; + } +} + +message TsRollingProto { + map tsRecords = 1; +} + +message SingleValueProto { + int64 ts = 1; + ObjectProto value = 2; +} + +message ObjectProto { + string type = 1; + string value = 2; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index e83a125265..20d0dda42f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -125,6 +125,7 @@ public class TbMsgAttributesNode implements TbNode { .scope(scope) .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) + .calculatedFieldIds(msg.getCalculatedFieldIds()) .callback(callback) .build()); } From 2dfbe2240d4bd22904e3d5edd6a7edeb15b0d1a5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 24 Dec 2024 16:25:01 +0200 Subject: [PATCH 063/416] added implementation to handle profile events when not my partition --- ...efaultCalculatedFieldExecutionService.java | 510 +++++++++--------- .../server/common/util/ProtoUtils.java | 40 ++ common/proto/src/main/proto/queue.proto | 14 +- 3 files changed, 306 insertions(+), 258 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index ff78d2925a..6e7536825a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -84,7 +84,6 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -98,6 +97,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; +import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; +import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; @TbCoreComponent @Service @@ -229,6 +230,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (!tpi.isMyPartition()) { + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldMsg(proto).build(), null); + log.debug("[{}][{}] Calculated field belongs to external partition. Probably rebalancing is in progress. Topic: {}", tenantId, calculatedFieldId, tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Calculated field belongs to external partition " + tpi.getFullTopicName() + "!")); + } if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); @@ -277,6 +284,54 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { + onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + } else { + callback.onSuccess(); + shouldReinit = false; + } + return shouldReinit; + } + + private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + cleanupEntity(calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + Set calculatedFieldIds = partitionedEntities.get(tpi); + if (calculatedFieldIds != null) { + calculatedFieldIds.remove(calculatedFieldId); + } + calculatedFieldCache.evict(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + List statesToRemove = states.keySet().stream() + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + rocksDBService.deleteAll(statesToRemove); + } catch (Exception e) { + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); + callback.onFailure(e); + } + } + + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; + } + @Override public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { try { @@ -288,8 +343,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId); - Optional.ofNullable(profileId).ifPresent(id -> calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); + List cfLinks = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> { + cfLinks.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); + }); cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); @@ -312,6 +369,23 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + } + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); @@ -334,75 +408,23 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - } - - private Object deserializeObjectProto(TransportProtos.ObjectProto objectProto) { - try { - String type = objectProto.getType(); - String value = objectProto.getValue(); - return switch (type) { - case "java.lang.String" -> value; - case "java.lang.Integer" -> Integer.parseInt(value); - case "java.lang.Long" -> Long.parseLong(value); - case "java.lang.Double" -> Double.parseDouble(value); - case "java.lang.Boolean" -> Boolean.parseBoolean(value); - default -> throw new IllegalArgumentException("Unsupported object type: " + type); - }; - } catch (Exception e) { - log.error("Failed to deserialize ObjectProto: [{}]", objectProto, e); - return null; - } - } - @Override public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - List calculatedFieldIds = new ArrayList<>(); - for (TransportProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { - CalculatedFieldId cfId = new CalculatedFieldId(new UUID( - cfIdProto.getCalculatedFieldIdMSB(), - cfIdProto.getCalculatedFieldIdLSB() - )); - calculatedFieldIds.add(cfId); + + if (proto.getClear()) { + clearState(tenantId, calculatedFieldId, entityId); + return; } - Map argumentsMap = new HashMap<>(); - proto.getArgumentsMap().forEach((key, entryProto) -> { - ArgumentEntry argumentEntry; - if (entryProto.hasTsRecords()) { - TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); - entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> { - Object value = deserializeObjectProto(objectProto); - tsRollingArgumentEntry.getTsRecords().put(ts, value); - }); - argumentEntry = tsRollingArgumentEntry; - } else if (entryProto.hasSingleValue()) { - TransportProtos.SingleValueProto singleRecordProto = entryProto.getSingleValue(); - Object value = deserializeObjectProto(singleRecordProto.getValue()); - argumentEntry = new SingleValueArgumentEntry(singleRecordProto.getTs(), value); - } else { - throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); - } - argumentsMap.put(key, argumentEntry); - }); + List calculatedFieldIds = proto.getCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList(); + Map argumentsMap = proto.getArgumentsMap().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); @@ -424,16 +446,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) - .forEach(cfId -> { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); - } - }); + .forEach(cfId -> clearState(tenantId, cfId, entityId)); initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); } catch (Exception e) { @@ -455,16 +468,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); - calculatedFieldIds.forEach(cfId -> { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); - } - }); + calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); @@ -475,94 +479,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { - TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - - if (argumentValues != null) { - argumentValues.forEach((key, argumentEntry) -> { - TransportProtos.ArgumentEntryProto.Builder argumentEntryProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); - - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); - - tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> { - TransportProtos.ObjectProto.Builder objectProtoBuilder = TransportProtos.ObjectProto.newBuilder() - .setType(value.getClass().getName()) - .setValue(value.toString()); - tsRollingProtoBuilder.putTsRecords(ts, objectProtoBuilder.build()); - }); - - argumentEntryProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - TransportProtos.SingleValueProto.Builder singleRecordProtoBuilder = TransportProtos.SingleValueProto.newBuilder() - .setTs(singleValueArgumentEntry.getTs()) - .setValue(TransportProtos.ObjectProto.newBuilder() - .setType(singleValueArgumentEntry.getValue().getClass().getName()) - .setValue(singleValueArgumentEntry.getValue().toString()) - .build()); - argumentEntryProtoBuilder.setSingleValue(singleRecordProtoBuilder.build()); - } - - msgBuilder.putArguments(key, argumentEntryProtoBuilder.build()); - }); - } - - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); - } - - private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (tpi.isMyPartition()) { + log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - callback.onSuccess(); - shouldReinit = false; + sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); } - return shouldReinit; - } - - private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - cleanupEntity(calculatedFieldId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - Set calculatedFieldIds = partitionedEntities.get(tpi); - if (calculatedFieldIds != null) { - calculatedFieldIds.remove(calculatedFieldId); - } - calculatedFieldCache.evict(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); - List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - rocksDBService.deleteAll(statesToRemove); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); - - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { @@ -605,82 +531,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - necessaryArguments.forEach((key, argument) -> { - futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - }); - return Futures.transform(Futures.allAsList(futures), results -> { - onComplete.accept(argumentValues); - return null; - }, calculatedFieldCallbackExecutor); - } - - private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = isProfileEntity(argumentEntityId) - ? targetEntityId - : argumentEntityId; - return fetchKvEntry(tenantId, entityId, argument); - } - - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - return switch (argument.getType()) { - case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); - case "ATTRIBUTE" -> transformSingleValueArgument( - Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), - calculatedFieldCallbackExecutor) - ); - case "TS_LATEST" -> transformSingleValueArgument( - Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), - calculatedFieldCallbackExecutor)); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - } - - private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { - long currentTime = System.currentTimeMillis(); - long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); - long startTs = currentTime - timeWindow; - int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); - ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - - return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); - } - - private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { - return Futures.transform(kvEntryFuture, kvEntry -> { - if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { - return ArgumentEntry.createSingleValueArgument(kvEntry.get()); - } else { - return SingleValueArgumentEntry.EMPTY; - } - }, calculatedFieldCallbackExecutor); - } - - private KvEntry createDefaultKvEntry(Argument argument) { - String key = argument.getKey(); - String defaultValue = argument.getDefaultValue(); - if (NumberUtils.isParsable(defaultValue)) { - return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); - } - if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { - return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); - } - return new StringDataEntry(key, defaultValue); - } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); @@ -742,14 +592,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, MoreExecutors.directExecutor()); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); - } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); - } - private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { String type = calculatedFieldResult.getType(); @@ -770,6 +612,166 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { + Map argumentValues = new HashMap<>(); + List> futures = new ArrayList<>(); + necessaryArguments.forEach((key, argument) -> { + futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + }); + return Futures.transform(Futures.allAsList(futures), results -> { + onComplete.accept(argumentValues); + return null; + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { + EntityId argumentEntityId = argument.getEntityId(); + EntityId entityId = isProfileEntity(argumentEntityId) + ? targetEntityId + : argumentEntityId; + return fetchKvEntry(tenantId, entityId, argument); + } + + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getType()) { + case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); + case "ATTRIBUTE" -> transformSingleValueArgument( + Futures.transform( + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor) + ); + case "TS_LATEST" -> transformSingleValueArgument( + Futures.transform( + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor)); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { + return Futures.transform(kvEntryFuture, kvEntry -> { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createSingleValueArgument(kvEntry.get()); + } else { + return SingleValueArgumentEntry.EMPTY; + } + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { + long currentTime = System.currentTimeMillis(); + long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); + long startTs = currentTime - timeWindow; + int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); + } + + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { + TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); + if (argumentValues != null) { + argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); + } + if (calculatedFieldIds != null) { + calculatedFieldIds.forEach(cfId -> msgBuilder.addCalculatedFields( + TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build() + )); + } + + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + } + + private void sendClearCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { + TransportProtos.CalculatedFieldStateMsgProto msg = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId) + .setClear(true) + .build(); + + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msg).build(), null); + } + + private TransportProtos.CalculatedFieldStateMsgProto.Builder createBaseCalculatedFieldStateMsg( + TenantId tenantId, + CalculatedFieldId calculatedFieldId, + EntityId entityId + ) { + return TransportProtos.CalculatedFieldStateMsgProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + } + + private TransportProtos.ArgumentEntryProto toArgumentEntryProto(ArgumentEntry argumentEntry) { + TransportProtos.ArgumentEntryProto.Builder argumentProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); + + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); + tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> + tsRollingProtoBuilder.putTsRecords(ts, toObjectProto(value)) + ); + argumentProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + argumentProtoBuilder.setSingleValue( + TransportProtos.SingleValueProto.newBuilder() + .setTs(singleValueArgumentEntry.getTs()) + .setValue(toObjectProto(singleValueArgumentEntry.getValue())) + .build() + ); + } + + return argumentProtoBuilder.build(); + } + + private ArgumentEntry fromArgumentEntryProto(TransportProtos.ArgumentEntryProto entryProto) { + if (entryProto.hasTsRecords()) { + TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); + entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> + tsRollingArgumentEntry.getTsRecords().put(ts, fromObjectProto(objectProto)) + ); + return tsRollingArgumentEntry; + } else if (entryProto.hasSingleValue()) { + TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); + return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue())); + } else { + throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); + } + } + + private KvEntry createDefaultKvEntry(Argument argument) { + String key = argument.getKey(); + String defaultValue = argument.getDefaultValue(); + if (NumberUtils.isParsable(defaultValue)) { + return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); + } + if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { + return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); + } + return new StringDataEntry(key, defaultValue); + } + + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); + } + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + } + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 264b23f118..ec17914fd8 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -1183,6 +1183,46 @@ public class ProtoUtils { return builder.build(); } + public static TransportProtos.ObjectProto toObjectProto(Object value) { + if (value == null) { + throw new IllegalArgumentException("Cannot convert null to ObjectProto"); + } + + TransportProtos.ObjectProto.Builder builder = TransportProtos.ObjectProto.newBuilder(); + + if (value instanceof String) { + builder.setStringValue((String) value); + } else if (value instanceof Integer) { + builder.setIntValue((Integer) value); + } else if (value instanceof Long) { + builder.setLongValue((Long) value); + } else if (value instanceof Double) { + builder.setDoubleValue((Double) value); + } else if (value instanceof Boolean) { + builder.setBoolValue((Boolean) value); + } else { + throw new IllegalArgumentException("Unsupported value type: " + value.getClass().getName()); + } + + return builder.build(); + } + + public static Object fromObjectProto(TransportProtos.ObjectProto proto) { + try { + return switch (proto.getValueCase()) { + case STRINGVALUE -> proto.getStringValue(); + case INTVALUE -> proto.getIntValue(); + case LONGVALUE -> proto.getLongValue(); + case DOUBLEVALUE -> proto.getDoubleValue(); + case BOOLVALUE -> proto.getBoolValue(); + case VALUE_NOT_SET -> throw new IllegalArgumentException("Value not set in ObjectProto"); + }; + } catch (Exception e) { + log.error("Failed to deserialize ObjectProto: [{}]", proto, e); + return null; + } + } + private static boolean isNotNull(Object obj) { return obj != null; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index f6f7afd5d8..581f9eb9f2 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -817,8 +817,9 @@ message CalculatedFieldStateMsgProto { string entityType = 5; int64 entityIdMSB = 6; int64 entityIdLSB = 7; - repeated CalculatedFieldIdProto calculatedFields = 8; - map arguments = 9; + bool clear = 8; + repeated CalculatedFieldIdProto calculatedFields = 9; + map arguments = 10; } message CalculatedFieldIdProto { @@ -843,8 +844,13 @@ message SingleValueProto { } message ObjectProto { - string type = 1; - string value = 2; + oneof value { + string stringValue = 1; + int32 intValue = 2; + int64 longValue = 3; + double doubleValue = 4; + bool boolValue = 5; + } } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From f6c5674e00d1675edba1e890f410d0d0ee2180bb Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Tue, 24 Dec 2024 16:32:45 +0200 Subject: [PATCH 064/416] Clean up Autowired imports --- .../service/edge/EdgeContextComponent.java | 60 ------------------- 1 file changed, 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index 9faafa6379..2ea64893d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -61,30 +61,15 @@ import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmProcessor; import org.thingsboard.server.service.edge.rpc.processor.alarm.comment.AlarmCommentProcessor; import org.thingsboard.server.service.edge.rpc.processor.asset.AssetEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.asset.profile.AssetProfileEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.customer.CustomerEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.dashboard.DashboardEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.device.DeviceEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.device.profile.DeviceProfileEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.edge.EdgeEntityProcessor; import org.thingsboard.server.service.edge.rpc.processor.entityview.EntityViewEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.notification.NotificationRuleEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.notification.NotificationTargetEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.notification.NotificationTemplateEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.oauth2.DomainEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.oauth2.OAuth2ClientEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.ota.OtaPackageEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.queue.QueueEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.relation.RelationEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.resource.ResourceEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.rule.RuleChainEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.rule.RuleChainMetadataEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.settings.AdminSettingsEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.telemetry.TelemetryEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.tenant.TenantEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.tenant.TenantProfileEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.user.UserEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.widget.WidgetBundleEdgeProcessor; -import org.thingsboard.server.service.edge.rpc.processor.widget.WidgetTypeEdgeProcessor; import org.thingsboard.server.service.edge.rpc.sync.EdgeRequestsService; import org.thingsboard.server.service.executors.GrpcCallbackExecutorService; @@ -213,9 +198,6 @@ public class EdgeContextComponent { // processors - @Autowired - private AdminSettingsEdgeProcessor adminSettingsProcessor; - @Autowired private AlarmProcessor alarmProcessor; @@ -228,9 +210,6 @@ public class EdgeContextComponent { @Autowired private AssetProfileEdgeProcessor assetProfileProcessor; - @Autowired - private CustomerEdgeProcessor customerProcessor; - @Autowired private DashboardEdgeProcessor dashboardProcessor; @@ -240,36 +219,12 @@ public class EdgeContextComponent { @Autowired private DeviceProfileEdgeProcessor deviceProfileProcessor; - @Autowired - private EdgeEntityProcessor edgeEntityProcessor; - @Autowired private EntityViewEdgeProcessor entityViewProcessor; @Autowired private NotificationRuleProcessor ruleProcessor; - @Autowired - private NotificationRuleEdgeProcessor notificationRuleProcessor; - - @Autowired - private NotificationTargetEdgeProcessor notificationTargetProcessor; - - @Autowired - private NotificationTemplateEdgeProcessor notificationTemplateProcessor; - - @Autowired - private DomainEdgeProcessor domainProcessor; - - @Autowired - private OAuth2ClientEdgeProcessor oAuth2ClientProcessor; - - @Autowired - private OtaPackageEdgeProcessor otaPackageProcessor; - - @Autowired - private QueueEdgeProcessor queueProcessor; - @Autowired private RelationEdgeProcessor relationProcessor; @@ -285,21 +240,6 @@ public class EdgeContextComponent { @Autowired private TelemetryEdgeProcessor telemetryProcessor; - @Autowired - private TenantEdgeProcessor tenantProcessor; - - @Autowired - private TenantProfileEdgeProcessor tenantProfileProcessor; - - @Autowired - private UserEdgeProcessor userProcessor; - - @Autowired - private WidgetBundleEdgeProcessor widgetBundleProcessor; - - @Autowired - private WidgetTypeEdgeProcessor widgetTypeProcessor; - // config @Autowired private EdgeEventStorageSettings edgeEventStorageSettings; From befd4cc9c69749dc510c5ff6ae2ff67713264d06 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 26 Dec 2024 10:56:20 +0200 Subject: [PATCH 065/416] added check for update values for ts rolling when rocksdb fails --- .../service/cf/DefaultCalculatedFieldCache.java | 8 ++++---- .../cf/DefaultCalculatedFieldExecutionService.java | 13 +++++++------ .../cf/ctx/state/BaseCalculatedFieldState.java | 1 + 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 1fc2a90e07..dd2ab3857c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -100,11 +100,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { calculatedFieldFetchLock.lock(); try { cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cfLinks != null) { calculatedFieldLinks.put(calculatedFieldId, cfLinks); @@ -122,11 +122,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { calculatedFieldFetchLock.lock(); try { cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); if (cfLinks != null) { entityIdCalculatedFieldLinks.put(entityId, cfLinks); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 6e7536825a..808c1ea8d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -556,20 +556,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldState state = calculatedFieldEntityCtx.getState(); boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - if (!allKeysPresent) { + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + + if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) .addListener(() -> performUpdateState.accept(state), calculatedFieldCallbackExecutor); - return; + } else { + performUpdateState.accept(state); } - performUpdateState.accept(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index b73ac51798..462fa19b17 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -24,6 +24,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected Map arguments; public BaseCalculatedFieldState() { + arguments = new HashMap<>(); } @Override From e787b805dae819f083876168e0a960bd5c801ae1 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 26 Dec 2024 17:05:27 +0200 Subject: [PATCH 066/416] added locking and logic to avoid iteration order issue in map --- ...efaultCalculatedFieldExecutionService.java | 68 +++++++++++-------- .../service/cf/ctx/state/ArgumentEntry.java | 5 +- .../ctx/state/BaseCalculatedFieldState.java | 49 ++++++------- .../cf/ctx/state/CalculatedFieldCtx.java | 8 ++- .../ctx/state/ScriptCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 14 ++-- .../cf/ctx/state/TsRollingArgumentEntry.java | 8 ++- common/proto/src/main/proto/queue.proto | 1 + 8 files changed, 87 insertions(+), 70 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 808c1ea8d8..af4b3e95e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -92,6 +92,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -119,6 +120,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; + private final ConcurrentMap entityLocks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -536,40 +539,47 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + ReentrantLock lock = entityLocks.computeIfAbsent(entityId, id -> new ReentrantLock()); + lock.lock(); + + try { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + } } - } - }; + }; - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); - if (!allKeysPresent || requiresTsRollingUpdate) { + if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + } else { + performUpdateState.accept(state); + } + } finally { + lock.unlock(); } } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); @@ -747,7 +757,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return tsRollingArgumentEntry; } else if (entryProto.hasSingleValue()) { TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); - return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue())); + return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue()), singleValueProto.getVersion()); } else { throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index ba7e094f77..dd56405352 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,8 +22,6 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; -import java.util.TreeMap; -import java.util.stream.Collectors; @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -48,8 +46,7 @@ public interface ArgumentEntry { } static ArgumentEntry createTsRollingArgument(List kvEntries) { - return new TsRollingArgumentEntry(kvEntries.stream(). - collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); + return new TsRollingArgumentEntry(kvEntries); } @JsonIgnore diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 462fa19b17..be7319b930 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -37,34 +36,30 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (arguments == null) { arguments = new HashMap<>(); } - AtomicBoolean stateUpdated = new AtomicBoolean(false); - argumentValues.forEach((key, argumentEntry) -> { - ArgumentEntry existingArgumentEntry = arguments.get(key); - if (existingArgumentEntry != null) { - if (existingArgumentEntry instanceof SingleValueArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - arguments.put(key, argumentEntry.copy()); - stateUpdated.set(true); - } - } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.addAllTsRecords(tsRollingArgumentEntry.getTsRecords()); - stateUpdated.set(true); - } - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.addTsRecord(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); - stateUpdated.set(true); - } - } + + boolean stateUpdated = false; + + for (Map.Entry entry : argumentValues.entrySet()) { + String key = entry.getKey(); + ArgumentEntry newEntry = entry.getValue(); + ArgumentEntry existingEntry = arguments.get(key); + + if (existingEntry == null || existingEntry.hasUpdatedValue(newEntry)) { + if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof TsRollingArgumentEntry newTsRollingEntry) { + existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); + } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { + existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); + } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry + && singleValueEntry.getVersion() > existingSingleValueEntry.getVersion()) { + arguments.put(key, newEntry.copy()); + } else { + arguments.put(key, newEntry.copy()); } - } else { - arguments.put(key, argumentEntry.copy()); - stateUpdated.set(true); + stateUpdated = true; } - }); - return stateUpdated.get(); + } + + return stateUpdated; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 2cd5c68144..d54a3220ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -26,6 +26,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.ArrayList; +import java.util.List; import java.util.Map; @Data @@ -36,6 +38,7 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; + private final List argKeys; private Output output; private String expression; private TbelInvokeService tbelInvokeService; @@ -48,10 +51,11 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); + this.argKeys = new ArrayList<>(arguments.keySet()); this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (!CalculatedFieldType.SIMPLE.equals(calculatedField.getType())) { + if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); } } @@ -65,7 +69,7 @@ public class CalculatedFieldCtx { tenantId, tbelInvokeService, expression, - arguments.keySet().toArray(new String[0]) + argKeys.toArray(String[]::new) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 87429050de..de7c514786 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -49,7 +49,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); } }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + Object[] args = ctx.getArgKeys().stream() + .map(key -> arguments.get(key).getValue()) + .toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); return Futures.transform(resultFuture, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 3f4fd5bdce..20b531f562 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -32,11 +32,15 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; + private long version; + public SingleValueArgumentEntry(KvEntry entry) { - if (entry instanceof TsKvEntry) { - this.ts = ((TsKvEntry) entry).getTs(); - } else if (entry instanceof AttributeKvEntry) { - this.ts = ((AttributeKvEntry) entry).getLastUpdateTs(); + if (entry instanceof TsKvEntry tsKvEntry) { + this.ts = tsKvEntry.getTs(); + this.version = tsKvEntry.getVersion(); + } else if (entry instanceof AttributeKvEntry attributeKvEntry) { + this.ts = attributeKvEntry.getLastUpdateTs(); + this.version = attributeKvEntry.getVersion(); } this.value = entry.getValue(); } @@ -66,7 +70,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { @Override public ArgumentEntry copy() { - return new SingleValueArgumentEntry(this.ts, this.value); + return new SingleValueArgumentEntry(this.ts, this.value, this.version); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 4ffa391550..64de2f8c8a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -16,16 +16,20 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import java.util.List; import java.util.Map; import java.util.TreeMap; @Data @NoArgsConstructor +@AllArgsConstructor @Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { @@ -35,8 +39,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords = new TreeMap<>(); - public TsRollingArgumentEntry(TreeMap tsRecords) { - addAllTsRecords(tsRecords); + public TsRollingArgumentEntry(List kvEntries) { + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry.getValue())); } /** diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 581f9eb9f2..1f66621a5b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -841,6 +841,7 @@ message TsRollingProto { message SingleValueProto { int64 ts = 1; ObjectProto value = 2; + int64 version = 3; } message ObjectProto { From 5f088a751e2fc18435fecfeee603c592ef412714 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 27 Dec 2024 18:20:20 +0200 Subject: [PATCH 067/416] refactored code --- .../cf/CalculatedFieldExecutionService.java | 9 +- .../service/cf/CalculatedFieldResult.java | 5 +- ...efaultCalculatedFieldExecutionService.java | 122 ++++++++++-------- .../service/cf/ctx/state/ArgumentEntry.java | 2 +- ...gumentType.java => ArgumentEntryType.java} | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 8 +- .../ctx/state/SingleValueArgumentEntry.java | 6 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 4 +- ...CalculatedFieldAttributeUpdateRequest.java | 38 ++++++ ...CalculatedFieldTelemetryUpdateRequest.java | 38 ++++++ ...alculatedFieldTimeSeriesUpdateRequest.java | 42 ++++++ .../DefaultTelemetrySubscriptionService.java | 6 +- .../CalculatedFieldControllerTest.java | 6 +- .../cf/CalculatedFieldLinkConfiguration.java | 4 +- .../data/cf/configuration/Argument.java | 2 +- .../data/cf/configuration/ArgumentType.java | 22 ++++ .../BaseCalculatedFieldConfiguration.java | 45 ++++--- .../common/data/cf/configuration/Output.java | 2 +- .../data/cf/configuration/OutputType.java | 22 ++++ .../server/dao/service/AssetServiceTest.java | 6 +- .../service/CalculatedFieldServiceTest.java | 6 +- .../dao/service/CustomerServiceTest.java | 6 +- .../server/dao/service/DeviceServiceTest.java | 6 +- 23 files changed, 302 insertions(+), 107 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{ArgumentType.java => ArgumentEntryType.java} (95%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index ee9b6d115c..e4b0a7ca1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,20 +15,15 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.List; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry); + void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index e8ea318bf6..52e0a0151b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -17,17 +17,18 @@ package org.thingsboard.server.service.cf; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import java.util.Map; @Data public final class CalculatedFieldResult { - private final String type; + private final OutputType type; private final AttributeScope scope; private final Map resultMap; - public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { + public CalculatedFieldResult(OutputType type, AttributeScope scope, Map resultMap) { this.type = type; this.scope = scope; this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index af4b3e95e5..27db9ae290 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,12 +35,15 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -48,7 +51,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -79,11 +81,13 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -92,7 +96,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -120,12 +123,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap entityLocks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; + private static final Set supportedReferencedEntities = EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT + ); + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -336,28 +341,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { + public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { try { - EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { - EntityId profileId = null; - if (EntityType.ASSET.equals(entityType)) { - profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else if (EntityType.DEVICE.equals(entityType)) { - profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - } - List cfLinks = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> { - cfLinks.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); - }); + TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); + AttributeScope scope = calculatedFieldTelemetryUpdateRequest.getScope(); + List telemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries(); + List calculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getCalculatedFieldIds(); + + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + EntityId profileId = getProfileId(tenantId, entityId); + + List cfLinks = Stream.concat( + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream(), + profileId != null ? calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream() : Stream.empty() + ).toList(); + cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map attributes = link.getConfiguration().getAttributes(); - Map timeSeries = link.getConfiguration().getTimeSeries(); + Map telemetryKeys = getTelemetryKeysFromLink(link, scope); Map updatedTelemetry = telemetry.stream() - .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .filter(entry -> telemetryKeys.containsValue(entry.getKey())) .collect(Collectors.toMap( - entry -> getMappedKey(entry, attributes, timeSeries), + entry -> getMappedKey(entry, telemetryKeys), entry -> entry, (v1, v2) -> v1 )); @@ -368,25 +374,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); } } catch (Exception e) { - log.trace("Failed to update telemetry entityId: [{}]", entityId, e); + log.trace("Failed to update telemetry.", e); } } - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); + private Map getTelemetryKeysFromLink(CalculatedFieldLink link, AttributeScope scope) { + return scope == null ? link.getConfiguration().getTimeSeries() : switch (scope) { + case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); + case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); + case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + }; + } + + private String getMappedKey(KvEntry entry, Map telemetry) { + return telemetry.entrySet().stream() + .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); } private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { @@ -539,17 +544,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { - ReentrantLock lock = entityLocks.computeIfAbsent(entityId, id -> new ReentrantLock()); - lock.lock(); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - try { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + states.compute(entityCtxId, (ctxId, ctx) -> { + CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); Consumer performUpdateState = (state) -> { if (state.updateState(argumentValues)) { calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && @@ -564,12 +566,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) @@ -578,9 +580,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { performUpdateState.accept(state); } - } finally { - lock.unlock(); - } + return calculatedFieldEntityCtx; + }); } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } @@ -605,9 +606,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { - String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + OutputType type = calculatedFieldResult.getType(); + TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); if (calculatedFieldIds == null) { calculatedFieldIds = new ArrayList<>(); @@ -649,19 +650,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { return switch (argument.getType()) { - case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); - case "ATTRIBUTE" -> transformSingleValueArgument( + case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); + case ATTRIBUTE -> transformSingleValueArgument( Futures.transform( attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), calculatedFieldCallbackExecutor) ); - case "TS_LATEST" -> transformSingleValueArgument( + case TS_LATEST -> transformSingleValueArgument( Futures.transform( timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), calculatedFieldCallbackExecutor)); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; } @@ -801,4 +801,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); } + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index dd56405352..b261840bfd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -35,7 +35,7 @@ import java.util.List; public interface ArgumentEntry { @JsonIgnore - ArgumentType getType(); + ArgumentEntryType getType(); Object getValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 360529a7e9..1a0dfb5ac7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -15,6 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state; -public enum ArgumentType { +public enum ArgumentEntryType { SINGLE_VALUE, TS_ROLLING } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index be7319b930..bc9a421e47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -49,8 +49,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); - } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry - && singleValueEntry.getVersion() > existingSingleValueEntry.getVersion()) { + } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { +// Long existingVersion = existingSingleValueEntry.getVersion(); +// Long newVersion = singleValueEntry.getVersion(); +// if (newVersion != null && (existingVersion == null || newVersion > existingVersion)) { +// arguments.put(key, newEntry.copy()); +// } arguments.put(key, newEntry.copy()); } else { arguments.put(key, newEntry.copy()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 20b531f562..8d5080e90f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -32,7 +32,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - private long version; + private Long version; public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry tsKvEntry) { @@ -54,8 +54,8 @@ public class SingleValueArgumentEntry implements ArgumentEntry { } @Override - public ArgumentType getType() { - return ArgumentType.SINGLE_VALUE; + public ArgumentEntryType getType() { + return ArgumentEntryType.SINGLE_VALUE; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 64de2f8c8a..1118e3af13 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -51,8 +51,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public ArgumentType getType() { - return ArgumentType.TS_ROLLING; + public ArgumentEntryType getType() { + return ArgumentEntryType.TS_ROLLING; } @JsonIgnore diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java new file mode 100644 index 0000000000..8479ff37d7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; + +import java.util.List; + +@Data +@AllArgsConstructor +public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { + + private TenantId tenantId; + private EntityId entityId; + private AttributeScope scope; + private List kvEntries; + private List calculatedFieldIds; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java new file mode 100644 index 0000000000..3c28833f31 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.List; + +public interface CalculatedFieldTelemetryUpdateRequest { + + TenantId getTenantId(); + + EntityId getEntityId(); + + AttributeScope getScope(); + + List getKvEntries(); + + List getCalculatedFieldIds(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java new file mode 100644 index 0000000000..987d899465 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +@AllArgsConstructor +public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { + + private TenantId tenantId; + private EntityId entityId; + private List kvEntries; + private List calculatedFieldIds; + + @Override + public AttributeScope getScope() { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index e35d3cead6..bf3076be5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -50,6 +50,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -152,7 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getCalculatedFieldIds(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getCalculatedFieldIds())); return saveFuture; } @@ -168,7 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getCalculatedFieldIds(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getCalculatedFieldIds())); } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index ba1dfb1fec..5d1467974d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -24,8 +24,10 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -140,7 +142,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -149,7 +151,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index c5f81cd572..78513c8b74 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -23,7 +23,9 @@ import java.util.Map; @Data public class CalculatedFieldLinkConfiguration { - private Map attributes = new HashMap<>(); + private Map clientAttributes = new HashMap<>(); + private Map serverAttributes = new HashMap<>(); + private Map sharedAttributes = new HashMap<>(); private Map timeSeries = new HashMap<>(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index f34f5e9cb7..4dac866219 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -24,7 +24,7 @@ public class Argument { private EntityId entityId; private String key; - private String type; + private ArgumentType type; private AttributeScope scope; private String defaultValue; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java new file mode 100644 index 0000000000..17e2315b52 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum ArgumentType { + + TS_LATEST, ATTRIBUTE, TS_ROLLING + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index ac36991a61..8c86b6c552 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -65,19 +65,24 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); - for (Map.Entry entry : arguments.entrySet()) { - Argument argument = entry.getValue(); - if (argument.getEntityId().equals(entityId)) { - switch (argument.getType()) { - case "ATTRIBUTE": - linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); - break; - case "TS_LATEST", "TS_ROLLING": - linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); - break; - } - } - } + arguments.entrySet().stream() + .filter(entry -> entry.getValue().getEntityId().equals(entityId)) + .forEach(entry -> { + Argument argument = entry.getValue(); + String argumentKey = entry.getKey(); + + switch (argument.getType()) { + case ATTRIBUTE -> { + switch (argument.getScope()) { + case CLIENT_SCOPE -> linkConfiguration.getClientAttributes().put(entry.getKey(), argument.getKey()); + case SERVER_SCOPE -> linkConfiguration.getServerAttributes().put(entry.getKey(), argument.getKey()); + case SHARED_SCOPE -> linkConfiguration.getSharedAttributes().put(entry.getKey(), argument.getKey()); + } + } + case TS_LATEST, TS_ROLLING -> + linkConfiguration.getTimeSeries().put(argumentKey, argument.getKey()); + } + }); return linkConfiguration; } @@ -98,7 +103,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("entityId", entityId.toString()); } argumentNode.put("key", argument.getKey()); - argumentNode.put("type", argument.getType()); + argumentNode.put("type", String.valueOf(argument.getType())); argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); @@ -112,7 +117,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (output != null) { ObjectNode outputNode = configNode.putObject("output"); outputNode.put("name", output.getName()); - outputNode.put("type", output.getType()); + outputNode.put("type", String.valueOf(output.getType())); if (output.getScope() != null) { outputNode.put("scope", String.valueOf(output.getScope())); } @@ -141,7 +146,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); } argument.setKey(argumentNode.get("key").asText()); - argument.setType(argumentNode.get("type").asText()); + JsonNode type = argumentNode.get("type"); + if (type != null && !type.isNull() && !type.asText().equals("null")) { + argument.setType(ArgumentType.valueOf(type.asText())); + } JsonNode scope = argumentNode.get("scope"); if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { argument.setScope(AttributeScope.valueOf(scope.asText())); @@ -169,7 +177,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (outputNode != null) { Output output = new Output(); output.setName(outputNode.get("name").asText()); - output.setType(outputNode.get("type").asText()); + JsonNode type = outputNode.get("type"); + if (type != null && !type.isNull() && !type.asText().equals("null")) { + output.setType(OutputType.valueOf(type.asText())); + } JsonNode scope = outputNode.get("scope"); if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { output.setScope(AttributeScope.valueOf(scope.asText())); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 46257d1ccc..12cf97338a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.AttributeScope; public class Output { private String name; - private String type; + private OutputType type; private AttributeScope scope; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java new file mode 100644 index 0000000000..c248bc8042 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum OutputType { + + TIME_SERIES, ATTRIBUTES + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index b0870f3dc5..9a0b9222f0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -33,7 +33,9 @@ import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -884,7 +886,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -893,7 +895,7 @@ public class AssetServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 9a1719e715..5a8f7a2383 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -26,8 +26,10 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -153,7 +155,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -162,7 +164,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 6671e0e821..d0ee833261 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -34,7 +34,9 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -379,7 +381,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -388,7 +390,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index f2f8686bc3..5b060ae145 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -42,7 +42,9 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1222,7 +1224,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -1231,7 +1233,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); From 8f87caba148aac97973b0d0f212d998957eb5a00 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 30 Dec 2024 16:50:01 +0200 Subject: [PATCH 068/416] implemented logic to handle profile updates even if no cf exists by them --- ...efaultCalculatedFieldExecutionService.java | 11 +++--- .../queue/DefaultTbClusterService.java | 36 ++++--------------- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 27db9ae290..96cbdfc73b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -542,6 +542,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); + Map argumentsMap = new HashMap<>(argumentValues); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -550,7 +551,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { + if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); @@ -564,17 +565,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) .addListener(() -> performUpdateState.accept(state), calculatedFieldCallbackExecutor); } else { @@ -583,7 +584,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFieldEntityCtx; }); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentsMap); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index d57a5266bf..6316d1fdb2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -68,7 +68,6 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -150,7 +149,6 @@ public class DefaultTbClusterService implements TbClusterService { private final GatewayNotificationsService gatewayNotificationsService; private final EdgeService edgeService; private final TbTransactionalCache edgeIdServiceIdCache; - private final CalculatedFieldService calculatedFieldService; @Override public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { @@ -393,7 +391,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - handleEntityDelete(tenantId, deviceId, device.getDeviceProfileId()); + sendProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -402,17 +400,10 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - handleEntityDelete(tenantId, assetId, asset.getAssetProfileId()); + sendProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), false, true); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } - private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); - if (cfExistsByProfile) { - sendProfileEntityEvent(tenantId, entityId, profileId, false, true); - } - } - @Override public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { onDeviceDeleted(oldTenantId, device, null); @@ -633,13 +624,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - handleProfileUpdate(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + sendEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { - handleEntityCreate(device.getTenantId(), device.getId(), device.getDeviceProfileId()); + sendProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -653,29 +644,14 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleProfileUpdate(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + sendEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } } else { - handleEntityCreate(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); + sendProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } - private void handleProfileUpdate(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - boolean cfExistsByOldProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); - boolean cfExistsByNewProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, newProfileId); - if (cfExistsByOldProfile || cfExistsByNewProfile) { - sendEntityProfileUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); - } - } - - private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); - if (cfExistsByProfile) { - sendProfileEntityEvent(tenantId, entityId, profileId, true, false); - } - } - @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { From e5003df778701a7191a577fb784d97cfc0c47eb4 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Mon, 30 Dec 2024 20:30:44 +0200 Subject: [PATCH 069/416] [WIP] save time series strategies: draft implementation for latest using Bloom filter --- .../DoNotSavePersistenceStrategy.java | 34 ++++++++++++ .../engine/telemetry/PersistenceStrategy.java | 35 ++++++++++++ .../SaveEveryMessagePersistenceStrategy.java | 34 ++++++++++++ ...aveFirstInIntervalPersistenceStrategy.java | 54 +++++++++++++++++++ .../engine/telemetry/TbMsgTimeseriesNode.java | 52 +++++++++++++++--- .../TbMsgTimeseriesNodeConfiguration.java | 12 ++++- .../telemetry/TbMsgTimeseriesNodeTest.java | 4 +- 7 files changed, 213 insertions(+), 12 deletions(-) create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/DoNotSavePersistenceStrategy.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/PersistenceStrategy.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveEveryMessagePersistenceStrategy.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveFirstInIntervalPersistenceStrategy.java diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/DoNotSavePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/DoNotSavePersistenceStrategy.java new file mode 100644 index 0000000000..5652a34cd1 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/DoNotSavePersistenceStrategy.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry; + +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.UUID; + +public final class DoNotSavePersistenceStrategy implements PersistenceStrategy { + + public static final DoNotSavePersistenceStrategy INSTANCE = new DoNotSavePersistenceStrategy(); + + private DoNotSavePersistenceStrategy() { + } + + @Override + public boolean shouldPersist(UUID originatorUuid, TsKvEntry timeseriesEntry) { + return false; + } + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/PersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/PersistenceStrategy.java new file mode 100644 index 0000000000..2aa92bd57c --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/PersistenceStrategy.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.UUID; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = SaveEveryMessagePersistenceStrategy.class, name = "SAVE_EVERY_MESSAGE"), + @JsonSubTypes.Type(value = SaveFirstInIntervalPersistenceStrategy.class, name = "SAVE_FIRST_IN_INTERVAL"), + @JsonSubTypes.Type(value = DoNotSavePersistenceStrategy.class, name = "DO_NOT_SAVE") +}) +public sealed interface PersistenceStrategy permits DoNotSavePersistenceStrategy, SaveEveryMessagePersistenceStrategy, SaveFirstInIntervalPersistenceStrategy { + + // TODO: maybe this should accept generic key? + boolean shouldPersist(UUID originatorUuid, TsKvEntry timeseriesEntry); + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveEveryMessagePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveEveryMessagePersistenceStrategy.java new file mode 100644 index 0000000000..7ca564b0bc --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveEveryMessagePersistenceStrategy.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry; + +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.UUID; + +public final class SaveEveryMessagePersistenceStrategy implements PersistenceStrategy { + + public static final SaveEveryMessagePersistenceStrategy INSTANCE = new SaveEveryMessagePersistenceStrategy(); + + private SaveEveryMessagePersistenceStrategy() { + } + + @Override + public boolean shouldPersist(UUID originatorUuid, TsKvEntry timeseriesEntry) { + return true; + } + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveFirstInIntervalPersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveFirstInIntervalPersistenceStrategy.java new file mode 100644 index 0000000000..4677058e6a --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveFirstInIntervalPersistenceStrategy.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.hash.BloomFilter; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +@SuppressWarnings("UnstableApiUsage") +public final class SaveFirstInIntervalPersistenceStrategy implements PersistenceStrategy { + + private final long intervalDurationMillis; + private final BloomFilter filter; + + @JsonCreator + public SaveFirstInIntervalPersistenceStrategy(@JsonProperty("intervalDurationMillis") long intervalDurationMillis) { + this.intervalDurationMillis = intervalDurationMillis; + // TODO: implement funnel as an enum + filter = BloomFilter.create((key, sink) -> + sink.putLong(key.intervalNumber()) + .putLong(key.originatorUuid().getMostSignificantBits()) + .putLong(key.originatorUuid().getLeastSignificantBits()) + .putString(key.timeseriesKey(), StandardCharsets.UTF_8), 1_000_000); + } + + // TODO: this should not be hardcoded here but should be defined by clients + // should be generified (what to do with funnel then?) + private record Key(long intervalNumber, UUID originatorUuid, String timeseriesKey) { + } + + @Override + public boolean shouldPersist(UUID originatorUuid, TsKvEntry timeseriesEntry) { + long intervalNumber = timeseriesEntry.getTs() / intervalDurationMillis; + return filter.put(new Key(intervalNumber, originatorUuid, timeseriesEntry.getKey())); + } + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 27f45feb47..28f9c88e34 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -15,8 +15,12 @@ */ package org.thingsboard.rule.engine.telemetry; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; @@ -27,6 +31,9 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -59,7 +66,7 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE "The DB layer has certain optimizations to ignore the updates of the \"attributes\" and \"latest values\" tables if the new record has a timestamp that is older than the previous record. " + "So, to make sure that all the messages will be processed correctly, one should enable this parameter for sequential message processing scenarios.", uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbActionNodeTimeseriesConfig", + // configDirective = "tbActionNodeTimeseriesConfig", icon = "file_upload" ) public class TbMsgTimeseriesNode implements TbNode { @@ -68,10 +75,13 @@ public class TbMsgTimeseriesNode implements TbNode { private TbContext ctx; private long tenantProfileDefaultStorageTtl; + private PersistenceStrategy latestPersistenceStrategy; + @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { this.config = TbNodeUtils.convert(configuration, TbMsgTimeseriesNodeConfiguration.class); this.ctx = ctx; + latestPersistenceStrategy = config.getPersistenceConfig().latest(); ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); } @@ -94,10 +104,18 @@ public class TbMsgTimeseriesNode implements TbNode { ctx.tellFailure(msg, new IllegalArgumentException("Msg body is empty: " + src)); return; } - List tsKvEntryList = new ArrayList<>(); + List withLatest = new ArrayList<>(); + List withoutLatest = new ArrayList<>(); for (Map.Entry> tsKvEntry : tsKvMap.entrySet()) { for (KvEntry kvEntry : tsKvEntry.getValue()) { - tsKvEntryList.add(new BasicTsKvEntry(tsKvEntry.getKey(), kvEntry)); + TsKvEntry entry = new BasicTsKvEntry(tsKvEntry.getKey(), kvEntry); + if (latestPersistenceStrategy.shouldPersist(msg.getOriginator().getId(), entry)) { + log.info("Persisting entry: {}", entry); + withLatest.add(entry); + } else { + log.info("Skipping entry: {}", entry); + withoutLatest.add(entry); + } } } String ttlValue = msg.getMetaData().getValue("TTL"); @@ -105,15 +123,33 @@ public class TbMsgTimeseriesNode implements TbNode { if (ttl == 0L) { ttl = tenantProfileDefaultStorageTtl; } - ctx.getTelemetryService().saveTimeseries(TimeseriesSaveRequest.builder() + + SettableFuture withLatestSavedFuture = SettableFuture.create(); + TimeseriesSaveRequest saveWithLatestRequest = TimeseriesSaveRequest.builder() + .tenantId(ctx.getTenantId()) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entries(withLatest) + .ttl(ttl) + .saveLatest(true) + .future(withLatestSavedFuture) + .build(); + ctx.getTelemetryService().saveTimeseries(saveWithLatestRequest); + + SettableFuture withoutLatestSavedFuture = SettableFuture.create(); + TimeseriesSaveRequest saveWithoutLatestRequest = TimeseriesSaveRequest.builder() .tenantId(ctx.getTenantId()) .customerId(msg.getCustomerId()) .entityId(msg.getOriginator()) - .entries(tsKvEntryList) + .entries(withoutLatest) .ttl(ttl) - .saveLatest(!config.isSkipLatestPersistence()) - .callback(new TelemetryNodeCallback(ctx, msg)) - .build()); + .saveLatest(false) + .future(withoutLatestSavedFuture) + .build(); + ctx.getTelemetryService().saveTimeseries(saveWithoutLatestRequest); + + ListenableFuture> bothSavedFuture = Futures.allAsList(withLatestSavedFuture, withoutLatestSavedFuture); + DonAsynchron.withCallback(bothSavedFuture, success -> ctx.tellSuccess(msg), failure -> ctx.tellFailure(msg, failure)); } public static long computeTs(TbMsg msg, boolean ignoreMetadataTs) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java index 1c33778a6b..adaebb0967 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.rule.engine.telemetry; +import lombok.Builder; import lombok.Data; import org.thingsboard.rule.engine.api.NodeConfiguration; @@ -22,15 +23,22 @@ import org.thingsboard.rule.engine.api.NodeConfiguration; public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration { private long defaultTTL; - private boolean skipLatestPersistence; private boolean useServerTs; + private PersistenceConfig persistenceConfig; @Override public TbMsgTimeseriesNodeConfiguration defaultConfiguration() { TbMsgTimeseriesNodeConfiguration configuration = new TbMsgTimeseriesNodeConfiguration(); configuration.setDefaultTTL(0L); - configuration.setSkipLatestPersistence(false); configuration.setUseServerTs(false); + configuration.setPersistenceConfig(PersistenceConfig.builder() + .latest(SaveEveryMessagePersistenceStrategy.INSTANCE) + .build()); return configuration; } + + @Builder + record PersistenceConfig(PersistenceStrategy latest) { + } + } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index 2cba4b8fb3..74240d3f97 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -88,7 +88,7 @@ public class TbMsgTimeseriesNodeTest { @Test public void verifyDefaultConfig() { assertThat(config.getDefaultTTL()).isEqualTo(0L); - assertThat(config.isSkipLatestPersistence()).isFalse(); + // assertThat(config.isSkipLatestPersistence()).isFalse(); assertThat(config.isUseServerTs()).isFalse(); } @@ -162,7 +162,7 @@ public class TbMsgTimeseriesNodeTest { public void givenSkipLatestPersistenceIsTrueAndTtlFromConfig_whenOnMsg_thenSaveTimeseriesUsingTtlFromConfig() throws TbNodeException { long ttlFromConfig = 5L; config.setDefaultTTL(ttlFromConfig); - config.setSkipLatestPersistence(true); + // config.setSkipLatestPersistence(true); init(); String data = """ From 63b79b7242d1c695cbacb46d156495247aa38ba8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 31 Dec 2024 16:15:42 +0200 Subject: [PATCH 070/416] added completable future to prevent reading old value from while updating is in progress --- .../DefaultCalculatedFieldExecutionService.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 96cbdfc73b..976139e8bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -94,6 +94,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; @@ -543,13 +544,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); states.compute(entityCtxId, (ctxId, ctx) -> { CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); + CompletableFuture updateFuture = new CompletableFuture<>(); + Consumer performUpdateState = (state) -> { if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); @@ -561,6 +566,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } } + updateFuture.complete(null); }; CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -570,7 +576,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -581,6 +586,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { performUpdateState.accept(state); } + + try { + updateFuture.join(); + } catch (Exception e) { + log.trace("Failed to update state for ctxId [{}].", ctxId, e); + throw new RuntimeException("Failed to update or initialize state.", e); + } + return calculatedFieldEntityCtx; }); } else { From 11cb43a115a6feb240497075430454aeaa12a475 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Tue, 31 Dec 2024 16:34:40 +0200 Subject: [PATCH 071/416] Redis-cache: relatedEdges changed key, lua improvement --- .../src/main/resources/thingsboard.yml | 2 +- .../cache/RedisTbTransactionalCache.java | 29 +++++++++++++++++++ .../server/cache/VersionedRedisTbCache.java | 28 ++---------------- .../cache/edge/RelatedEdgesCacheKey.java | 2 +- 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 066c1468fb..32fd62670d 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -600,7 +600,7 @@ cache: maxSize: "${CACHE_SPECS_EDGE_SESSIONS_MAX_SIZE:10000}" # 0 means the cache is disabled relatedEdges: timeToLiveInMinutes: "${CACHE_SPECS_RELATED_EDGES_TTL:1440}" # Related Edges cache TTL - maxSize: "${CACHE_SPECS_RELATED_EDGES_MAX_SIZE:0}" # 0 means the cache is disabled + maxSize: "${CACHE_SPECS_RELATED_EDGES_MAX_SIZE:10000}" # 0 means the cache is disabled repositorySettings: timeToLiveInMinutes: "${CACHE_SPECS_REPOSITORY_SETTINGS_TTL:1440}" # Repository settings cache TTL maxSize: "${CACHE_SPECS_REPOSITORY_SETTINGS_MAX_SIZE:10000}" # 0 means the cache is disabled diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index 8ee58ab8eb..1eee03ccf9 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -19,9 +19,11 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.support.NullValue; +import org.springframework.dao.InvalidDataAccessApiUsageException; import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.connection.RedisStringCommands; +import org.springframework.data.redis.connection.ReturnType; import org.springframework.data.redis.connection.jedis.JedisClusterConnection; import org.springframework.data.redis.connection.jedis.JedisConnection; import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; @@ -269,4 +271,31 @@ public abstract class RedisTbTransactionalCache extends RedisTbTransactionalCache implements VersionedTbCache { @@ -67,15 +65,7 @@ public abstract class VersionedRedisTbCache Date: Fri, 3 Jan 2025 16:00:35 +0200 Subject: [PATCH 072/416] handled profile updates in cluster --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 25 +++++++++++ ...efaultCalculatedFieldExecutionService.java | 44 +++++++++--------- .../queue/DefaultTbClusterService.java | 26 ++++++----- .../queue/DefaultTbCoreConsumerService.java | 38 ++++++++++++++-- .../queue/DefaultTbEdgeConsumerService.java | 8 ++-- .../DefaultTbRuleEngineConsumerService.java | 6 ++- .../processing/AbstractConsumerService.java | 9 ++++ common/proto/src/main/proto/queue.proto | 2 + docker/docker-compose.cluster.yml | 45 +++++++++++++++++++ 10 files changed, 166 insertions(+), 39 deletions(-) create mode 100644 docker/docker-compose.cluster.yml diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index f953e57cc3..ad683c324c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -34,6 +34,8 @@ public interface CalculatedFieldCache { List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index dd2ab3857c..a4077b599b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -141,6 +141,31 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return cfLinks; } + + @Override + public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); + calculatedFieldFetchLock.lock(); + try { + List cfLinks = getCalculatedFieldLinks(tenantId, calculatedFieldId); + if (cfLinks != null && !cfLinks.isEmpty()) { + cfLinks.forEach(link -> { + entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { + if (existingList == null) { + existingList = new ArrayList<>(); + } else if (!(existingList instanceof ArrayList)) { + existingList = new ArrayList<>(existingList); + } + existingList.add(link); + return existingList; + }); + }); + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + @Override public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 976139e8bd..2f55de6b77 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -71,7 +71,6 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; @@ -105,7 +104,6 @@ import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; -@TbCoreComponent @Service @Slf4j @RequiredArgsConstructor @@ -117,7 +115,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; + // private final RocksDBService rocksDBService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; @@ -213,8 +211,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void restoreState(CalculatedField cf, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId()); - String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); +// String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + String storedState = null; if (storedState != null) { CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); states.put(ctxId, restoredCtx); @@ -313,13 +312,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldIds != null) { calculatedFieldIds.remove(calculatedFieldId); } - calculatedFieldCache.evict(calculatedFieldId); +// calculatedFieldCache.evict(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) .map(JacksonUtil::writeValueAsString) .toList(); - rocksDBService.deleteAll(statesToRemove); +// rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); @@ -414,7 +413,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds); } - log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } @Override @@ -423,7 +421,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - + log.info("Received CalculatedFieldStateMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}], entityId=[{}]", tenantId, calculatedFieldId, entityId); if (proto.getClear()) { clearState(tenantId, calculatedFieldId, entityId); return; @@ -431,7 +429,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas List calculatedFieldIds = proto.getCalculatedFieldsList().stream() .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList(); + .collect(Collectors.toCollection(ArrayList::new)); Map argumentsMap = proto.getArgumentsMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); @@ -451,8 +449,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> clearState(tenantId, cfId, entityId)); @@ -472,15 +470,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); + + List calculatedFieldIds = Stream.concat( calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); + calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); } } catch (Exception e) { @@ -494,7 +495,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); +// rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); } @@ -558,13 +559,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Consumer performUpdateState = (state) -> { if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); +// rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); if (allArgsPresent) { performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } + log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); } updateFuture.complete(null); }; @@ -633,6 +635,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldIds.add(calculatedFieldId); TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); + log.info("Pushed message to rule engine: originatorId=[{}]", originatorId); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); } @@ -715,6 +718,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas )); } + log.info("Sending calculated field state msg from entityId [{}]", entityId); clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } @@ -790,11 +794,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); - } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); +// String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); +// if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); +// } +// return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 6316d1fdb2..551b0ddbc2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -391,7 +391,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - sendProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); + handleProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -400,7 +400,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - sendProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), false, true); + handleProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), true, true); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } @@ -563,7 +563,9 @@ public class DefaultTbClusterService implements TbClusterService { || entityType.equals(EntityType.API_USAGE_STATE) || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) || entityType.equals(EntityType.ENTITY_VIEW) - || entityType.equals(EntityType.NOTIFICATION_RULE)) { + || entityType.equals(EntityType.NOTIFICATION_RULE) + || entityType.equals(EntityType.CALCULATED_FIELD) + ) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { @@ -624,13 +626,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - sendEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + handleEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { - sendProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); + handleProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -644,10 +646,10 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - sendEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + handleEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } } else { - sendProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); + handleProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @@ -792,8 +794,8 @@ public class DefaultTbClusterService implements TbClusterService { public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { CalculatedFieldId calculatedFieldId = calculatedField.getId(); broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); - sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); } private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { @@ -809,7 +811,7 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } - private void sendEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -822,10 +824,12 @@ public class DefaultTbClusterService implements TbClusterService { builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); TransportProtos.EntityProfileUpdateMsgProto msg = builder.build(); + + broadcastToCore(ToCoreNotificationMsg.newBuilder().setEntityProfileUpdateMsg(msg).build()); pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } - private void sendProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { + private void handleProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { TransportProtos.ProfileEntityMsgProto.Builder builder = TransportProtos.ProfileEntityMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -838,6 +842,8 @@ public class DefaultTbClusterService implements TbClusterService { builder.setAdded(added); builder.setDeleted(deleted); TransportProtos.ProfileEntityMsgProto msg = builder.build(); + + broadcastToCore(ToCoreNotificationMsg.newBuilder().setProfileEntityMsg(msg).build()); pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setProfileEntityMsg(msg).build(), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 1be25b308f..6baa75b3ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; @@ -87,6 +88,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; @@ -109,6 +111,7 @@ import org.thingsboard.server.service.ws.notification.sub.NotificationRequestUpd import org.thingsboard.server.service.ws.notification.sub.NotificationUpdate; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -181,8 +184,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); + if (added) { + entitiesByProfile.add(entityId); + } else { + entitiesByProfile.remove(entityId); + } + } + private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { if (msg.hasSubEvent()) { TbEntitySubEventProto subEvent = msg.getSubEvent(); @@ -688,12 +718,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { - log.warn("[{}] Failed to process device type updated message for device [{}]", tenantId.getId(), entityId.getId(), t); + log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); callback.onFailure(t); }); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java index f219d7ae69..35c7894f2f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java @@ -91,7 +91,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService future = edgeCtx.getTenantProfileProcessor().processEntityNotification(tenantId, edgeNotificationMsg); case NOTIFICATION_RULE, NOTIFICATION_TARGET, NOTIFICATION_TEMPLATE -> future = edgeCtx.getNotificationEdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case TB_RESOURCE -> future = edgeCtx.getResourceProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case DOMAIN, OAUTH2_CLIENT -> future = edgeCtx.getOAuth2EdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); + case TB_RESOURCE -> + future = edgeCtx.getResourceProcessor().processEntityNotification(tenantId, edgeNotificationMsg); + case DOMAIN, OAUTH2_CLIENT -> + future = edgeCtx.getOAuth2EdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); default -> { future = Futures.immediateFuture(null); log.warn("[{}] Edge event type [{}] is not designed to be pushed to edge", tenantId, type); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 73810949b3..7d4d975cb4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -46,6 +46,7 @@ import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; @@ -83,8 +84,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TbApiUsageStateService apiUsageStateService, PartitionService partitionService, ApplicationEventPublisher eventPublisher, - JwtSettingsService jwtSettingsService) { - super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); + JwtSettingsService jwtSettingsService, + CalculatedFieldCache calculatedFieldCache) { + super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.ctx = ctx; this.tbDeviceRpcService = tbDeviceRpcService; this.queueService = queueService; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 95fc8cb843..2aaed13ec1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -43,6 +44,7 @@ import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.TbPackCallback; @@ -68,6 +70,7 @@ public abstract class AbstractConsumerService Date: Mon, 6 Jan 2025 14:15:32 +0200 Subject: [PATCH 073/416] changed partititoning implementation --- ...efaultCalculatedFieldExecutionService.java | 180 ++++++++---------- ...CalculatedFieldAttributeUpdateRequest.java | 13 +- ...CalculatedFieldTelemetryUpdateRequest.java | 9 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 9 +- .../DefaultTelemetrySubscriptionService.java | 20 +- .../thingsboard/server/common/msg/TbMsg.java | 32 ++-- common/proto/src/main/proto/queue.proto | 2 +- docker/docker-compose.cluster.yml | 45 ----- .../engine/api/AttributesSaveRequest.java | 10 +- .../engine/api/TimeseriesSaveRequest.java | 10 +- .../engine/telemetry/TbMsgAttributesNode.java | 2 +- .../engine/telemetry/TbMsgTimeseriesNode.java | 2 +- 12 files changed, 145 insertions(+), 189 deletions(-) delete mode 100644 docker/docker-compose.cluster.yml diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 2f55de6b77..f7bcb4e678 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,7 +35,6 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -98,7 +97,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; @@ -115,7 +113,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - // private final RocksDBService rocksDBService; + private final RocksDBService rocksDBService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; @@ -168,35 +166,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - Map> tpiCalculatedFieldMap = new HashMap<>(); + Map> tpiTargetEntityMap = new HashMap<>(); for (CalculatedField cf : cfs) { - TopicPartitionInfo tpi; - try { - tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); - } catch (Exception e) { - log.warn("Failed to resolve partition for CalculatedField [{}], tenant id [{}]. Reason: {}", - cf.getId(), cf.getTenantId(), e.getMessage()); - continue; - } - if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { - tpiCalculatedFieldMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(cf); + + Consumer resolvePartition = entityId -> { + TopicPartitionInfo tpi; + try { + tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { + tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId())); + } + } catch (Exception e) { + log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", + entityId, cf.getTenantId(), e.getMessage()); + } + }; + + EntityId cfEntityId = cf.getEntityId(); + if (isProfileEntity(cfEntityId)) { + calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); + } else { + resolvePartition.accept(cfEntityId); } } - for (var entry : tpiCalculatedFieldMap.entrySet()) { - for (List partition : Lists.partition(entry.getValue(), 1000)) { + for (var entry : tpiTargetEntityMap.entrySet()) { + for (List partition : Lists.partition(entry.getValue(), 1000)) { log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); var future = calculatedFieldExecutor.submit(() -> { try { - for (CalculatedField cf : partition) { - EntityId cfEntityId = cf.getEntityId(); - if (isProfileEntity(cfEntityId)) { - calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId) - .forEach(entityId -> restoreState(cf, entityId)); - } else { - restoreState(cf, cfEntityId); - } + for (CalculatedFieldEntityCtxId ctxId : partition) { + restoreState(ctxId.cfId(), ctxId.entityId()); } } catch (Throwable t) { log.error("Unexpected exception while restoring CalculatedField states", t); @@ -209,17 +210,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return result; } - private void restoreState(CalculatedField cf, EntityId entityId) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId()); -// String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + private void restoreState(UUID calculatedFieldId, UUID entityId) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); + String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); - String storedState = null; if (storedState != null) { CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); states.put(ctxId, restoredCtx); - log.info("Restored state for CalculatedField [{}]", cf.getId()); + log.info("Restored state for CalculatedField [{}]", calculatedFieldId); } else { - log.warn("No state found for CalculatedField [{}], entity [{}].", cf.getId(), entityId); + log.warn("No state found for CalculatedField [{}], entity [{}].", calculatedFieldId, entityId); } } @@ -238,12 +238,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - if (!tpi.isMyPartition()) { - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldMsg(proto).build(), null); - log.debug("[{}][{}] Calculated field belongs to external partition. Probably rebalancing is in progress. Topic: {}", tenantId, calculatedFieldId, tpi.getFullTopicName()); - callback.onFailure(new RuntimeException("Calculated field belongs to external partition " + tpi.getFullTopicName() + "!")); - } if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); @@ -307,18 +301,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - Set calculatedFieldIds = partitionedEntities.get(tpi); - if (calculatedFieldIds != null) { - calculatedFieldIds.remove(calculatedFieldId); - } -// calculatedFieldCache.evict(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) .map(JacksonUtil::writeValueAsString) .toList(); -// rocksDBService.deleteAll(statesToRemove); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); @@ -345,22 +333,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); - AttributeScope scope = calculatedFieldTelemetryUpdateRequest.getScope(); - List telemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries(); - List calculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getCalculatedFieldIds(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { EntityId profileId = getProfileId(tenantId, entityId); - List cfLinks = Stream.concat( - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream(), - profileId != null ? calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream() : Stream.empty() - ).toList(); - - cfLinks.forEach(link -> { + getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = getTelemetryKeysFromLink(link, scope); - Map updatedTelemetry = telemetry.stream() + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link); + Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() .filter(entry -> telemetryKeys.containsValue(entry.getKey())) .collect(Collectors.toMap( entry -> getMappedKey(entry, telemetryKeys), @@ -369,7 +349,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas )); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, calculatedFieldIds, updatedTelemetry); + List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); } }); } @@ -378,14 +359,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private Map getTelemetryKeysFromLink(CalculatedFieldLink link, AttributeScope scope) { - return scope == null ? link.getConfiguration().getTimeSeries() : switch (scope) { - case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); - case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); - case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); - }; - } - private String getMappedKey(KvEntry entry, Map telemetry) { return telemetry.entrySet().stream() .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) @@ -394,7 +367,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .orElse(entry.getKey()); } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); @@ -406,12 +379,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); + calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, previousCalculatedFieldIds)); } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, previousCalculatedFieldIds); } } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds); + default -> + updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, previousCalculatedFieldIds); } } @@ -427,14 +401,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return; } - List calculatedFieldIds = proto.getCalculatedFieldsList().stream() + List previousCalculatedFieldIds = proto.getPreviousCalculatedFieldsList().stream() .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) .collect(Collectors.toCollection(ArrayList::new)); Map argumentsMap = proto.getArgumentsMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); } @@ -449,9 +423,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> clearState(tenantId, cfId, entityId)); @@ -470,18 +441,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); - - - List calculatedFieldIds = Stream.concat( - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) - ).toList(); - calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId)); + getCalculatedFieldLinks(tenantId, entityId, profileId) + .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); } } catch (Exception e) { @@ -490,12 +454,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); states.remove(ctxId); -// rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); } @@ -541,12 +505,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -559,12 +523,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Consumer performUpdateState = (state) -> { if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); -// rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); } log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); } @@ -599,17 +563,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFieldEntityCtx; }); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentsMap); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, previousCalculatedFieldIds, argumentsMap); } } - private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List previousCalculatedFieldIds) { ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, calculatedFieldIds); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, previousCalculatedFieldIds); } } @@ -620,20 +584,20 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, MoreExecutors.directExecutor()); } - private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { + private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List previousCalculatedFieldIds) { try { OutputType type = calculatedFieldResult.getType(); TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); - if (calculatedFieldIds == null) { - calculatedFieldIds = new ArrayList<>(); - } - if (calculatedFieldIds.contains(calculatedFieldId)) { + if (previousCalculatedFieldIds != null && previousCalculatedFieldIds.contains(calculatedFieldId)) { throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); } + List calculatedFieldIds = previousCalculatedFieldIds != null + ? new ArrayList<>(previousCalculatedFieldIds) + : new ArrayList<>(); calculatedFieldIds.add(calculatedFieldId); - TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).previousCalculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); log.info("Pushed message to rule engine: originatorId=[{}]", originatorId); } catch (Exception e) { @@ -641,6 +605,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private List getCalculatedFieldLinks(TenantId tenantId, EntityId entityId, EntityId profileId) { + List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + if (profileId != null) { + links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId)); + } + return links; + } + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { Map argumentValues = new HashMap<>(); List> futures = new ArrayList<>(); @@ -704,13 +676,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List previousCalculatedFieldIds, Map argumentValues) { TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); if (argumentValues != null) { argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); } - if (calculatedFieldIds != null) { - calculatedFieldIds.forEach(cfId -> msgBuilder.addCalculatedFields( + if (previousCalculatedFieldIds != null) { + previousCalculatedFieldIds.forEach(cfId -> msgBuilder.addPreviousCalculatedFields( TransportProtos.CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) @@ -794,11 +766,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { -// String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); -// if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); -// } -// return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); + } + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 8479ff37d7..c56217b2ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -18,12 +18,14 @@ package org.thingsboard.server.service.cf.telemetry; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import java.util.List; +import java.util.Map; @Data @AllArgsConstructor @@ -33,6 +35,15 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel private EntityId entityId; private AttributeScope scope; private List kvEntries; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; + + @Override + public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { + return switch (scope) { + case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); + case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); + case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + }; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 3c28833f31..98062a08db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,13 +15,14 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; +import java.util.Map; public interface CalculatedFieldTelemetryUpdateRequest { @@ -29,10 +30,10 @@ public interface CalculatedFieldTelemetryUpdateRequest { EntityId getEntityId(); - AttributeScope getScope(); - List getKvEntries(); - List getCalculatedFieldIds(); + List getPreviousCalculatedFieldIds(); + + Map getTelemetryKeysFromLink(CalculatedFieldLink link); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 987d899465..bd2161dca1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -17,13 +17,14 @@ package org.thingsboard.server.service.cf.telemetry; import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.Map; @Data @AllArgsConstructor @@ -32,11 +33,11 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe private TenantId tenantId; private EntityId entityId; private List kvEntries; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; @Override - public AttributeScope getScope() { - return null; + public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { + return link.getConfiguration().getTimeSeries(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index bf3076be5c..99cbd89494 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -154,7 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getCalculatedFieldIds())); + addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds()))); return saveFuture; } @@ -170,7 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getCalculatedFieldIds())); + addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds()))); } @Override @@ -243,7 +243,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .onlyLatest(true) .callback(new FutureCallback<>() { @Override - public void onSuccess(@Nullable Void tmp) {} + public void onSuccess(@Nullable Void tmp) { + } @Override public void onFailure(Throwable t) { @@ -342,4 +343,17 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer }; } + protected void addCalculatedFieldCallback(ListenableFuture saveFuture, Consumer callback) { + Futures.addCallback(saveFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable T result) { + callback.accept(result); + } + + @Override + public void onFailure(Throwable t) { + } + }, tsCallBackExecutor); + } + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 0805175f77..9d99f38c60 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -35,10 +35,10 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; /** * Created by ashvayka on 13.01.18. @@ -67,7 +67,7 @@ public final class TbMsg implements Serializable { private final UUID correlationId; private final Integer partition; - private final List calculatedFieldIds; + private final List previousCalculatedFieldIds; @Getter(value = AccessLevel.NONE) @JsonIgnore @@ -117,7 +117,7 @@ public final class TbMsg implements Serializable { } private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List calculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List previousCalculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { this.id = id != null ? id : UUID.randomUUID(); this.queueName = queueName; if (ts > 0) { @@ -144,7 +144,9 @@ public final class TbMsg implements Serializable { this.ruleNodeId = ruleNodeId; this.correlationId = correlationId; this.partition = partition; - this.calculatedFieldIds = calculatedFieldIds; + this.previousCalculatedFieldIds = previousCalculatedFieldIds != null + ? new CopyOnWriteArrayList<>(previousCalculatedFieldIds) + : new CopyOnWriteArrayList<>(); this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx(); this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } @@ -192,8 +194,8 @@ public final class TbMsg implements Serializable { builder.setPartition(msg.getPartition()); } - if (msg.getCalculatedFieldIds() != null) { - for (CalculatedFieldId calculatedFieldId : msg.getCalculatedFieldIds()) { + if (msg.getPreviousCalculatedFieldIds() != null) { + for (CalculatedFieldId calculatedFieldId : msg.getPreviousCalculatedFieldIds()) { MsgProtos.CalculatedFieldIdProto calculatedFieldIdProto = MsgProtos.CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) @@ -216,7 +218,7 @@ public final class TbMsg implements Serializable { RuleNodeId ruleNodeId = null; UUID correlationId = null; Integer partition = null; - List calculatedFieldIds = new ArrayList<>(); + List calculatedFieldIds = new CopyOnWriteArrayList<>(); if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); } @@ -274,6 +276,7 @@ public final class TbMsg implements Serializable { /** * Checks if the message is still valid for processing. May be invalid if the message pack is timed-out or canceled. + * * @return 'true' if message is valid for processing, 'false' otherwise. */ public boolean isValid() { @@ -368,7 +371,7 @@ public final class TbMsg implements Serializable { protected RuleNodeId ruleNodeId; protected UUID correlationId; protected Integer partition; - protected List calculatedFieldIds; + protected List previousCalculatedFieldIds; protected TbMsgProcessingCtx ctx; protected TbMsgCallback callback; @@ -390,7 +393,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = tbMsg.ruleNodeId; this.correlationId = tbMsg.correlationId; this.partition = tbMsg.partition; - this.calculatedFieldIds = tbMsg.calculatedFieldIds; + this.previousCalculatedFieldIds = tbMsg.previousCalculatedFieldIds; this.ctx = tbMsg.ctx; this.callback = tbMsg.callback; } @@ -413,8 +416,7 @@ public final class TbMsg implements Serializable { /** *

Deprecated: This should only be used when you need to specify a custom message type that doesn't exist in the {@link TbMsgType} enum. * Prefer using {@link #type(TbMsgType)} instead. - * - * */ + */ @Deprecated public TbMsgBuilder type(String type) { this.type = type; @@ -482,8 +484,8 @@ public final class TbMsg implements Serializable { return this; } - public TbMsgBuilder calculatedFieldIds(List calculatedFieldIds) { - this.calculatedFieldIds = calculatedFieldIds; + public TbMsgBuilder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; return this; } @@ -498,7 +500,7 @@ public final class TbMsg implements Serializable { } public TbMsg build() { - return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); + return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, previousCalculatedFieldIds, ctx, callback); } public String toString() { @@ -506,7 +508,7 @@ public final class TbMsg implements Serializable { ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + - ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", calculatedFields=" + this.calculatedFieldIds + + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds + ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1c4927c422..9cba62eecc 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -818,7 +818,7 @@ message CalculatedFieldStateMsgProto { int64 entityIdMSB = 6; int64 entityIdLSB = 7; bool clear = 8; - repeated CalculatedFieldIdProto calculatedFields = 9; + repeated CalculatedFieldIdProto previousCalculatedFields = 9; map arguments = 10; } diff --git a/docker/docker-compose.cluster.yml b/docker/docker-compose.cluster.yml deleted file mode 100644 index 4c74238a10..0000000000 --- a/docker/docker-compose.cluster.yml +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright © 2016-2024 The Thingsboard Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: '3.0' - -services: - kafka: - restart: always - image: "bitnami/kafka:3.7.0" - ports: - - "9092:9092" - env_file: - - kafka.env - depends_on: - - zookeeper - zookeeper: - restart: always - image: "zookeeper:3.8.0" - ports: - - "2181" - environment: - ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 - ZOO_ADMINSERVER_ENABLED: "false" - redis: - restart: always - image: bitnami/redis:7.2 - environment: - # ALLOW_EMPTY_PASSWORD is recommended only for development. - ALLOW_EMPTY_PASSWORD: "yes" - ports: - - '6379:6379' diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index 9747a6033e..406b2d0d5c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -41,7 +41,7 @@ public class AttributesSaveRequest { private final AttributeScope scope; private final List entries; private final boolean notifyDevice; - private final List calculatedFieldIds; + private final List previousCalculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -55,7 +55,7 @@ public class AttributesSaveRequest { private AttributeScope scope; private List entries; private boolean notifyDevice = true; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; private FutureCallback callback; Builder() {} @@ -103,8 +103,8 @@ public class AttributesSaveRequest { return this; } - public Builder calculatedFieldIds(List calculatedFieldIds) { - this.calculatedFieldIds = calculatedFieldIds; + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; return this; } @@ -128,7 +128,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, calculatedFieldIds, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, previousCalculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 12afa2d939..95eb788e5f 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -41,7 +41,7 @@ public class TimeseriesSaveRequest { private final long ttl; private final boolean saveLatest; private final boolean onlyLatest; - private final List calculatedFieldIds; + private final List previousCalculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -58,7 +58,7 @@ public class TimeseriesSaveRequest { private FutureCallback callback; private boolean saveLatest = true; private boolean onlyLatest; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; Builder() {} @@ -106,8 +106,8 @@ public class TimeseriesSaveRequest { return this; } - public Builder calculatedFieldIds(List calculatedFieldIds) { - this.calculatedFieldIds = calculatedFieldIds; + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; return this; } @@ -131,7 +131,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, calculatedFieldIds, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, previousCalculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 20d0dda42f..ae2fce6575 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -125,7 +125,7 @@ public class TbMsgAttributesNode implements TbNode { .scope(scope) .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) - .calculatedFieldIds(msg.getCalculatedFieldIds()) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .callback(callback) .build()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 386e56320a..89f7844313 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -112,7 +112,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entries(tsKvEntryList) .ttl(ttl) .saveLatest(!config.isSkipLatestPersistence()) - .calculatedFieldIds(msg.getCalculatedFieldIds()) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } From 7d8a76ce7f8e1ca15e08709bb2fcba8446578fbf Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 6 Jan 2025 15:32:23 +0200 Subject: [PATCH 074/416] Draft of the review --- .../cf/DefaultCalculatedFieldCache.java | 10 ++++------ .../AbstractSubscriptionService.java | 9 +++++++-- .../DefaultTelemetrySubscriptionService.java | 20 +++++-------------- common/proto/src/main/proto/queue.proto | 7 +++++++ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index a4077b599b..c4293b9edc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -122,16 +122,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { calculatedFieldFetchLock.lock(); try { cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); - if (cfLinks != null) { - entityIdCalculatedFieldLinks.put(entityId, cfLinks); - log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); - } + entityIdCalculatedFieldLinks.put(entityId, cfLinks); + log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); } } finally { calculatedFieldFetchLock.unlock(); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index e26ede2bae..0bb8f76398 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.service.subscription.SubscriptionManagerService; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -99,7 +100,11 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList } protected void addWsCallback(ListenableFuture saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback() { + addCallback(saveFuture, callback, wsCallBackExecutor); + } + + protected void addCallback(ListenableFuture saveFuture, Consumer callback, Executor executor) { + Futures.addCallback(saveFuture, new FutureCallback<>() { @Override public void onSuccess(@Nullable T result) { callback.accept(result); @@ -108,7 +113,7 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList @Override public void onFailure(Throwable t) { } - }, wsCallBackExecutor); + }, executor); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 99cbd89494..f873e48774 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -154,7 +154,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds()))); + // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. + //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); return saveFuture; } @@ -170,7 +172,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds()))); + //CalculatedFieldAttributeUpdateRequest - add constructor that accepts the AttributesSaveRequest + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); } @Override @@ -343,17 +346,4 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer }; } - protected void addCalculatedFieldCallback(ListenableFuture saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback() { - @Override - public void onSuccess(@Nullable T result) { - callback.accept(result); - } - - @Override - public void onFailure(Throwable t) { - } - }, tsCallBackExecutor); - } - } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 9cba62eecc..56f347f311 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -809,6 +809,13 @@ message ProfileEntityMsgProto { bool deleted = 10; } +message ToServerB { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + repeated CfIdEntityIdPair links = 3; + value = 4; +} + message CalculatedFieldStateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; From e2ac2708b658c3802c9f15f9825bb621a3730f85 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 6 Jan 2025 17:37:33 +0200 Subject: [PATCH 075/416] changed CF links config --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 37 ++++++++++++++----- ...efaultCalculatedFieldExecutionService.java | 28 +++++++++++++- ...CalculatedFieldAttributeUpdateRequest.java | 21 +++++++---- ...CalculatedFieldTelemetryUpdateRequest.java | 4 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 16 +++++--- .../DefaultTelemetrySubscriptionService.java | 7 +--- .../server/dao/cf/CalculatedFieldService.java | 2 + .../BaseCalculatedFieldConfiguration.java | 17 +++++---- .../dao/cf/BaseCalculatedFieldService.java | 7 ++++ .../server/dao/cf/CalculatedFieldDao.java | 2 + .../dao/sql/cf/CalculatedFieldRepository.java | 3 ++ .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 +++ 13 files changed, 114 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index ad683c324c..aa25565a34 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -30,6 +30,8 @@ public interface CalculatedFieldCache { CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index c4293b9edc..40d569b040 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -56,6 +56,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final DeviceService deviceService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); @@ -65,10 +66,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Getter private int initFetchPackSize; - @PostConstruct public void init() { - // to discuss: fetch on start or fetch on demand PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); @@ -97,19 +96,37 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return calculatedField; } + @Override + public List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + List cfs = entityIdCalculatedFields.get(entityId); + if (cfs == null) { + calculatedFieldFetchLock.lock(); + try { + cfs = entityIdCalculatedFields.get(entityId); + if (cfs == null) { + cfs = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId, entityId); + entityIdCalculatedFields.put(entityId, cfs); + log.debug("[{}] Fetch calculated fields by entity into cache: {}", entityId, cfs); + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated fields by entity in cache: {}", entityId, cfs); + return cfs; + } + @Override public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { calculatedFieldFetchLock.lock(); try { cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - if (cfLinks != null) { - calculatedFieldLinks.put(calculatedFieldId, cfLinks); - log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); - } + calculatedFieldLinks.put(calculatedFieldId, cfLinks); + log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); } } finally { calculatedFieldFetchLock.unlock(); @@ -139,7 +156,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return cfLinks; } - @Override public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); @@ -225,6 +241,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldLinks.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); + entityIdCalculatedFields.remove(oldCalculatedField.getEntityId()); log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldsCtx.remove(calculatedFieldId); log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f7bcb4e678..c0f7acf0a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -139,6 +140,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(() -> rocksDBService.getAll() + .forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class)))); } @PreDestroy @@ -337,11 +340,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (supportedReferencedEntities.contains(entityId.getEntityType())) { EntityId profileId = getProfileId(tenantId, entityId); + // process by profile + if (profileId != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, profileId).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(profileId); + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(linkConfiguration); + Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) + .collect(Collectors.toMap( + entry -> getMappedKey(entry, telemetryKeys), + entry -> entry, + (v1, v2) -> v1 + )); + + if (!updatedTelemetry.isEmpty()) { + List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, cf.getId(), previousCalculatedFieldIds, updatedTelemetry); + } + }); + } + + // process by links getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link); + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link.getConfiguration()); Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsValue(entry.getKey())) + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) .collect(Collectors.toMap( entry -> getMappedKey(entry, telemetryKeys), entry -> entry, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index c56217b2ce..25d2f57bd6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -15,10 +15,10 @@ */ package org.thingsboard.server.service.cf.telemetry; -import lombok.AllArgsConstructor; import lombok.Data; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; @Data -@AllArgsConstructor public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; @@ -37,12 +36,20 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel private List kvEntries; private List previousCalculatedFieldIds; + public CalculatedFieldAttributeUpdateRequest(AttributesSaveRequest request) { + this.tenantId = request.getTenantId(); + this.entityId = request.getEntityId(); + this.scope = request.getScope(); + this.kvEntries = request.getEntries(); + this.previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + } + @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { + public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { return switch (scope) { - case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); - case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); - case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + case CLIENT_SCOPE -> linkConfiguration.getClientAttributes(); + case SERVER_SCOPE -> linkConfiguration.getServerAttributes(); + case SHARED_SCOPE -> linkConfiguration.getSharedAttributes(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 98062a08db..29ee899ec9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,6 +34,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { List getPreviousCalculatedFieldIds(); - Map getTelemetryKeysFromLink(CalculatedFieldLink link); + Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index bd2161dca1..6225286631 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.service.cf.telemetry; -import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; @Data -@AllArgsConstructor public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; @@ -35,9 +34,16 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe private List kvEntries; private List previousCalculatedFieldIds; + public CalculatedFieldTimeSeriesUpdateRequest(TimeseriesSaveRequest request) { + this.tenantId = request.getTenantId(); + this.entityId = request.getEntityId(); + this.kvEntries = request.getEntries(); + this.previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + } + @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { - return link.getConfiguration().getTimeSeries(); + public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { + return linkConfiguration.getTimeSeries(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f873e48774..8773564e5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -154,9 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. - //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)), tsCallBackExecutor); return saveFuture; } @@ -172,8 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - //CalculatedFieldAttributeUpdateRequest - add constructor that accepts the AttributesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } @Override diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 1e64fdac60..3a508a5c08 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -38,6 +38,8 @@ public interface CalculatedFieldService extends EntityDaoService { List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 8c86b6c552..c3c4e32507 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -68,19 +68,22 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel arguments.entrySet().stream() .filter(entry -> entry.getValue().getEntityId().equals(entityId)) .forEach(entry -> { - Argument argument = entry.getValue(); + Argument tergetArgument = entry.getValue(); String argumentKey = entry.getKey(); - switch (argument.getType()) { + switch (tergetArgument.getType()) { case ATTRIBUTE -> { - switch (argument.getScope()) { - case CLIENT_SCOPE -> linkConfiguration.getClientAttributes().put(entry.getKey(), argument.getKey()); - case SERVER_SCOPE -> linkConfiguration.getServerAttributes().put(entry.getKey(), argument.getKey()); - case SHARED_SCOPE -> linkConfiguration.getSharedAttributes().put(entry.getKey(), argument.getKey()); + switch (tergetArgument.getScope()) { + case CLIENT_SCOPE -> + linkConfiguration.getClientAttributes().put(tergetArgument.getKey(), argumentKey); + case SERVER_SCOPE -> + linkConfiguration.getServerAttributes().put(tergetArgument.getKey(), argumentKey); + case SHARED_SCOPE -> + linkConfiguration.getSharedAttributes().put(tergetArgument.getKey(), argumentKey); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(argumentKey, argument.getKey()); + linkConfiguration.getTimeSeries().put(tergetArgument.getKey(), argumentKey); } }); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0849414a0e..e650aec35e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -98,6 +98,13 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findCalculatedFieldIdsByEntityId(tenantId, entityId); } + @Override + public List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldsByEntityId(tenantId, entityId); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 5b3bcc2750..39663d0afc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -31,6 +31,8 @@ public interface CalculatedFieldDao extends Dao { List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 9aa0aee428..816fa1546c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -28,6 +29,8 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index e3762f6157..20081299e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -55,6 +55,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); From 03c3341265724341aea3668f6f376a7e90d842f3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 8 Jan 2025 12:39:49 +0200 Subject: [PATCH 076/416] added logic to send msgs to RE when not my partition --- .../server/controller/BaseController.java | 7 - .../cf/CalculatedFieldExecutionService.java | 2 + ...efaultCalculatedFieldExecutionService.java | 252 ++++++++++++++---- .../cf/ctx/CalculatedFieldEntityCtxId.java | 5 +- ...CalculatedFieldAttributeUpdateRequest.java | 6 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 6 +- .../TbRuleEngineQueueConsumerManager.java | 9 +- .../BaseCalculatedFieldConfiguration.java | 14 +- .../server/common/util/ProtoUtils.java | 133 +++++++++ common/proto/src/main/proto/queue.proto | 29 +- .../dao/cf/BaseCalculatedFieldService.java | 1 + 11 files changed, 386 insertions(+), 78 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 4987096d17..139c61d710 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -366,9 +366,6 @@ public abstract class BaseController { @Autowired protected TbServiceInfoProvider serviceInfoProvider; - @Autowired - protected CalculatedFieldService calculatedFieldService; - @Autowired protected NotificationTargetService notificationTargetService; @@ -998,10 +995,6 @@ public abstract class BaseController { return null; } - protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { - return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation); - } - protected MediaType parseMediaType(String contentType) { try { return MediaType.parseMediaType(contentType); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index e4b0a7ca1e..6d1d459b9b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -25,6 +25,8 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); + void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); + void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c0f7acf0a3..0de3136439 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,7 +35,9 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; @@ -51,6 +53,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -67,6 +70,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -80,7 +84,9 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -102,6 +108,7 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @Slf4j @@ -177,8 +184,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi; try { tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); - if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { - tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId())); + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { + tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); } } catch (Exception e) { log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", @@ -213,7 +220,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return result; } - private void restoreState(UUID calculatedFieldId, UUID entityId) { + private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); @@ -232,7 +239,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); } @Override @@ -243,7 +250,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); + onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); @@ -293,7 +300,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { callback.onSuccess(); shouldReinit = false; @@ -301,12 +308,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return shouldReinit; } - private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId)) .map(JacksonUtil::writeValueAsString) .toList(); rocksDBService.deleteAll(statesToRemove); @@ -334,61 +341,147 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { try { - TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { - EntityId profileId = getProfileId(tenantId, entityId); - - // process by profile - if (profileId != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, profileId).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(profileId); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(linkConfiguration); - Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, telemetryKeys), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, cf.getId(), previousCalculatedFieldIds, updatedTelemetry); - } + TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + Map> tpiStatesToUpdate = new HashMap<>(); + + updateTelemetryForEntity(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + updateTelemetryForProfile(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId), tpiStatesToUpdate); + updateTelemetryForLinkedEntities(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); }); } + } + } catch (Exception e) { + log.trace("Failed to update telemetry.", e); + } + } - // process by links - getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { + private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + updateTelemetryForEntity(request, request.getEntityId(), tpiStates); + } + + private void updateTelemetryForProfile(CalculatedFieldTelemetryUpdateRequest request, EntityId profileId, Map> tpiStates) { + updateTelemetryForEntity(request, profileId, tpiStates); + } + + private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + if (targetEntity != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(targetEntity); + mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); + }); + } + } else { + List ctxIds = tpiStates.computeIfAbsent(tpi, k -> new ArrayList<>()); + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { + ctxIds.add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); + }); + } + } + + private void updateTelemetryForLinkedEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + if (targetEntityTpi.isMyPartition()) { + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + } else { + List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); + ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); + } + } + + private void updateTelemetryForLinkedEntities(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId) + .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link.getConfiguration()); - Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, telemetryKeys), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); + EntityId targetEntityId = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId).getEntityId(); + + if (isProfileEntity(targetEntityId)) { + calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { + updateTelemetryForLinkedEntity(request, entityByProfile, link, tpiStates); + }); + } else { + updateTelemetryForLinkedEntity(request, targetEntityId, link, tpiStates); } }); - } - } catch (Exception e) { - log.trace("Failed to update telemetry.", e); + } + + private void mapAndProcessUpdatedTelemetry(TenantId tenantId, + EntityId entityId, + CalculatedFieldId calculatedFieldId, + CalculatedFieldTelemetryUpdateRequest request, + CalculatedFieldLinkConfiguration linkConfiguration) { + Map telemetryKeys = request.getTelemetryKeysFromLink(linkConfiguration); + Map updatedTelemetry = mapTelemetryKeys(telemetryKeys, request.getKvEntries()); + + if (!updatedTelemetry.isEmpty()) { + List previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); } } - private String getMappedKey(KvEntry entry, Map telemetry) { - return telemetry.entrySet().stream() - .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); + private Map mapTelemetryKeys(Map telemetryKeys, List kvEntries) { + return kvEntries.stream() + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) + .collect(Collectors.toMap( + entry -> telemetryKeys.getOrDefault(entry.getKey(), entry.getKey()), + entry -> entry, + (v1, v2) -> v1 + )); + } + + @Override + public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + + proto.getLinksList().forEach(ctxIdProto -> { + EntityId entityId = EntityIdFactory.getByTypeAndUuid( + ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + + List updatedTelemetry = proto.getUpdatedTelemetryList().stream() + .map(ProtoUtils::fromTelemetryProto) + .toList(); + + boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); + + CalculatedFieldTelemetryUpdateRequest request = attributesUpdated + ? new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()) + : new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + + onTelemetryUpdate(request); + }); + } catch (Exception e) { + log.trace("Failed to process telemetry update msg: [{}]", proto, e); + } } private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { @@ -481,7 +574,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { @@ -537,7 +630,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); states.compute(entityCtxId, (ctxId, ctx) -> { CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); @@ -777,6 +870,57 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( + CalculatedFieldTelemetryUpdateRequest request, List links + ) { + TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); + + builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()); + + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + + for (CalculatedFieldId calculatedFieldId : request.getPreviousCalculatedFieldIds()) { + builder.addPreviousCalculatedFields(toProto(calculatedFieldId)); + } + + if (request instanceof CalculatedFieldAttributeUpdateRequest attributeUpdateRequest) { + builder.setScope(attributeUpdateRequest.getScope().name()); + } + + for (KvEntry entry : request.getKvEntries()) { + TransportProtos.TelemetryProto.Builder telemetryBuilder = TransportProtos.TelemetryProto.newBuilder(); + if (request instanceof CalculatedFieldTimeSeriesUpdateRequest) { + telemetryBuilder.setTsKv(toTsKvProto((TsKvEntry) entry)); + } + if (request instanceof CalculatedFieldAttributeUpdateRequest attrRequest) { + telemetryBuilder.setAttrKv(ProtoUtils.toAttributeKvProto((AttributeKvEntry) entry, attrRequest.getScope())); + } + builder.addUpdatedTelemetry(telemetryBuilder.build()); + } + + return builder.build(); + } + + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getKey(); String defaultValue = argument.getDefaultValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index f7c451efee..5fb90a3e46 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -15,7 +15,8 @@ */ package org.thingsboard.server.service.cf.ctx; -import java.util.UUID; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; -public record CalculatedFieldEntityCtxId(UUID cfId, UUID entityId) { +public record CalculatedFieldEntityCtxId(CalculatedFieldId cfId, EntityId entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 25d2f57bd6..a83cc0fc25 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.telemetry; +import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; @@ -22,18 +23,19 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; private EntityId entityId; private AttributeScope scope; - private List kvEntries; + private List kvEntries; private List previousCalculatedFieldIds; public CalculatedFieldAttributeUpdateRequest(AttributesSaveRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 6225286631..507daf386e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -15,23 +15,25 @@ */ package org.thingsboard.server.service.cf.telemetry; +import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; private EntityId entityId; - private List kvEntries; + private List kvEntries; private List previousCalculatedFieldIds; public CalculatedFieldTimeSeriesUpdateRequest(TimeseriesSaveRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index c2823d3c00..243a3adbf7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -34,6 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; @@ -63,14 +64,18 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager entry.getValue().getEntityId().equals(entityId)) .forEach(entry -> { - Argument tergetArgument = entry.getValue(); + Argument targetArgument = entry.getValue(); String argumentKey = entry.getKey(); - switch (tergetArgument.getType()) { + switch (targetArgument.getType()) { case ATTRIBUTE -> { - switch (tergetArgument.getScope()) { + switch (targetArgument.getScope()) { case CLIENT_SCOPE -> - linkConfiguration.getClientAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getClientAttributes().put(targetArgument.getKey(), argumentKey); case SERVER_SCOPE -> - linkConfiguration.getServerAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getServerAttributes().put(targetArgument.getKey(), argumentKey); case SHARED_SCOPE -> - linkConfiguration.getSharedAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getSharedAttributes().put(targetArgument.getKey(), argumentKey); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getTimeSeries().put(targetArgument.getKey(), argumentKey); } }); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index ec17914fd8..d332bac64f 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -58,12 +59,14 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; @@ -627,6 +630,136 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static KvEntry fromProto(TransportProtos.TsKvProto proto) { + TransportProtos.KeyValueProto kvProto = proto.getKv(); + String key = kvProto.getKey(); + KvEntry entry = switch (kvProto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); + case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); + case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); + default -> null; + }; + return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); + } + + public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { + if (telemetryProto.hasAttrKv()) { + return fromProto(telemetryProto.getAttrKv().getValue()); + } else if (telemetryProto.hasTsKv()) { + return fromProto(telemetryProto.getTsKv()); + } else { + throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); + } + } + + public static TransportProtos.AttributeKey toAttributeKeyProto(String key, AttributeScope scope) { + TransportProtos.AttributeKey.Builder builder = TransportProtos.AttributeKey.newBuilder(); + builder.setAttributeKey(key); + switch (scope) { + case CLIENT_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.CLIENT_SCOPE); + break; + case SERVER_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.SERVER_SCOPE); + break; + case SHARED_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.SHARED_SCOPE); + break; + default: + throw new IllegalArgumentException("Unsupported attribute scope: " + scope); + } + return builder.build(); + } + + public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { + return TransportProtos.AttributeKvProto.newBuilder() + .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) + .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) + .build(); + } + + public static TransportProtos.AttributeValueProto toAttributeValueProto(AttributeKvEntry attributeKvEntry) { + TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder(); + builder.setLastUpdateTs(attributeKvEntry.getLastUpdateTs()); + switch (attributeKvEntry.getDataType()) { + case BOOLEAN: + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) + .setHasV(true) + .setBoolV(attributeKvEntry.getBooleanValue().orElse(false)); + break; + case LONG: + builder.setType(TransportProtos.KeyValueType.LONG_V) + .setHasV(true) + .setLongV(attributeKvEntry.getLongValue().orElse(0L)); + break; + case DOUBLE: + builder.setType(TransportProtos.KeyValueType.DOUBLE_V) + .setHasV(true) + .setDoubleV(attributeKvEntry.getDoubleValue().orElse(0.0)); + break; + case STRING: + builder.setType(TransportProtos.KeyValueType.STRING_V) + .setHasV(true) + .setStringV(attributeKvEntry.getStrValue().orElse("")); + break; + case JSON: + builder.setType(TransportProtos.KeyValueType.JSON_V) + .setHasV(true) + .setJsonV(attributeKvEntry.getJsonValue().orElse("{}")); + break; + default: + builder.setHasV(false); + throw new IllegalArgumentException("Unsupported AttributeKvEntry data type: " + attributeKvEntry.getDataType()); + } + if (attributeKvEntry.getKey() != null) { + builder.setKey(attributeKvEntry.getKey()); + } + if (attributeKvEntry.getVersion() != null) { + builder.setVersion(attributeKvEntry.getVersion()); + } + return builder.build(); + } + + public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { + return TransportProtos.TsKvProto.newBuilder() + .setTs(tsKvEntry.getTs()) + .setKv(toKeyValueProto(tsKvEntry)) + .setVersion(tsKvEntry.getVersion()) + .build(); + } + + public static TransportProtos.KeyValueProto toKeyValueProto(KvEntry kvEntry) { + TransportProtos.KeyValueProto.Builder builder = TransportProtos.KeyValueProto.newBuilder(); + builder.setKey(kvEntry.getKey()); + switch (kvEntry.getDataType()) { + case BOOLEAN: + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) + .setBoolV(kvEntry.getBooleanValue().orElse(false)); + break; + case LONG: + builder.setType(TransportProtos.KeyValueType.LONG_V) + .setLongV(kvEntry.getLongValue().orElse(0L)); + break; + case DOUBLE: + builder.setType(TransportProtos.KeyValueType.DOUBLE_V) + .setDoubleV(kvEntry.getDoubleValue().orElse(0.0)); + break; + case STRING: + builder.setType(TransportProtos.KeyValueType.STRING_V) + .setStringV(kvEntry.getStrValue().orElse("")); + break; + case JSON: + builder.setType(TransportProtos.KeyValueType.JSON_V) + .setJsonV(kvEntry.getJsonValue().orElse("{}")); + break; + default: + throw new IllegalArgumentException("Unsupported KvEntry data type: " + kvEntry.getDataType()); + } + return builder.build(); + } + public static TransportProtos.DeviceProto toProto(Device device) { var builder = TransportProtos.DeviceProto.newBuilder() .setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 56f347f311..685ca47719 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -183,6 +183,18 @@ message TsKvListProto { repeated KeyValueProto kv = 2; } +message AttributeKvProto { + AttributeKey key = 1; + AttributeValueProto value = 2; +} + +message TelemetryProto { + oneof proto { + AttributeKvProto attrKv = 1; + TsKvProto tsKv = 2; + } +} + message DeviceInfoProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -809,11 +821,21 @@ message ProfileEntityMsgProto { bool deleted = 10; } -message ToServerB { +message TelemetryUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - repeated CfIdEntityIdPair links = 3; - value = 4; + repeated CalculatedFieldEntityCtxIdProto links = 3; + repeated CalculatedFieldIdProto previousCalculatedFields = 4; + string scope = 5; + repeated TelemetryProto updatedTelemetry = 6; +} + +message CalculatedFieldEntityCtxIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; } message CalculatedFieldStateMsgProto { @@ -1655,6 +1677,7 @@ message ToRuleEngineMsg { bytes tbMsg = 3; repeated string relationTypes = 4; string failureMessage = 5; + TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; } message ToRuleEngineNotificationMsg { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index e650aec35e..36bc3d038a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -247,6 +247,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); return cfConfig.getReferencedEntities().stream() + .filter(referencedEntity -> !referencedEntity.equals(calculatedField.getEntityId())) .map(referencedEntityId -> { CalculatedFieldLink link = new CalculatedFieldLink(); link.setTenantId(tenantId); From 46180e33d70761ea18f93238420e8ed1cf97f31e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 8 Jan 2025 17:20:20 +0200 Subject: [PATCH 077/416] cache refactoring --- .../service/cf/CalculatedFieldCache.java | 16 +- .../cf/DefaultCalculatedFieldCache.java | 132 +++++++-------- ...efaultCalculatedFieldExecutionService.java | 156 ++++++++++-------- .../processing/AbstractConsumerService.java | 4 +- .../BaseCalculatedFieldConfiguration.java | 21 +++ .../CalculatedFieldConfiguration.java | 7 + common/proto/src/main/proto/queue.proto | 11 +- .../dao/cf/BaseCalculatedFieldService.java | 18 +- .../dao/sql/cf/CalculatedFieldRepository.java | 3 +- .../dao/sql/cf/JpaCalculatedFieldDao.java | 2 +- 10 files changed, 195 insertions(+), 175 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index aa25565a34..7394c95f08 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -28,20 +28,24 @@ import java.util.Set; public interface CalculatedFieldCache { - CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldsByEntityId(EntityId entityId); - List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldLinksByEntityId(EntityId entityId); - void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + void evict(CalculatedFieldId calculatedFieldId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 40d569b040..71fc3eff67 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -70,98 +71,44 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { public void init() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + calculatedFields.values().forEach(cf -> + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cf) + ); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + calculatedFieldLinks.values().stream() + .flatMap(List::stream) + .forEach(link -> + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link) + ); } @Override - public CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - CalculatedField calculatedField = calculatedFields.get(calculatedFieldId); - if (calculatedField == null) { - calculatedFieldFetchLock.lock(); - try { - calculatedField = calculatedFields.get(calculatedFieldId); - if (calculatedField == null) { - calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (calculatedField != null) { - calculatedFields.put(calculatedFieldId, calculatedField); - log.debug("[{}] Fetch calculated field into cache: {}", calculatedFieldId, calculatedField); - } - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field in cache: {}", calculatedFieldId, calculatedField); - return calculatedField; + public CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId) { + return calculatedFields.get(calculatedFieldId); } @Override - public List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { - List cfs = entityIdCalculatedFields.get(entityId); - if (cfs == null) { - calculatedFieldFetchLock.lock(); - try { - cfs = entityIdCalculatedFields.get(entityId); - if (cfs == null) { - cfs = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId, entityId); - entityIdCalculatedFields.put(entityId, cfs); - log.debug("[{}] Fetch calculated fields by entity into cache: {}", entityId, cfs); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated fields by entity in cache: {}", entityId, cfs); - return cfs; + public List getCalculatedFieldsByEntityId(EntityId entityId) { + return entityIdCalculatedFields.getOrDefault(entityId, new ArrayList<>()); } @Override - public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { - calculatedFieldFetchLock.lock(); - try { - cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { - cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - calculatedFieldLinks.put(calculatedFieldId, cfLinks); - log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field links in cache: {}", calculatedFieldId, cfLinks); - return cfLinks; + public List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { + return calculatedFieldLinks.getOrDefault(calculatedFieldId, new ArrayList<>()); } @Override - public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { - List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { - calculatedFieldFetchLock.lock(); - try { - cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { - cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); - entityIdCalculatedFieldLinks.put(entityId, cfLinks); - log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field links by entity id in cache: {}", entityId, cfLinks); - return cfLinks; + public List getCalculatedFieldLinksByEntityId(EntityId entityId) { + return entityIdCalculatedFieldLinks.getOrDefault(entityId, new ArrayList<>()); } @Override - public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + public void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); calculatedFieldFetchLock.lock(); try { - List cfLinks = getCalculatedFieldLinks(tenantId, calculatedFieldId); + List cfLinks = getCalculatedFieldLinks(calculatedFieldId); if (cfLinks != null && !cfLinks.isEmpty()) { cfLinks.forEach(link -> { entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { @@ -181,14 +128,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { calculatedFieldFetchLock.lock(); try { ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { - CalculatedField calculatedField = getCalculatedField(tenantId, calculatedFieldId); + CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); calculatedFieldsCtx.put(calculatedFieldId, ctx); @@ -236,6 +183,42 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entities; } + @Override + public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + calculatedFieldFetchLock.lock(); + try { + CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + EntityId cfEntityId = calculatedField.getEntityId(); + + calculatedFields.put(calculatedFieldId, calculatedField); + + entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new ArrayList<>()).add(calculatedField); + + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); + + configuration.getReferencedEntities().stream() + .filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) + .forEach(referencedEntityId -> { + entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new ArrayList<>()) + .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); + }); + } finally { + calculatedFieldFetchLock.unlock(); + } + } + + @Override + public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + calculatedFieldFetchLock.lock(); + try { + evict(calculatedFieldId); + addCalculatedField(tenantId, calculatedFieldId); + } finally { + calculatedFieldFetchLock.unlock(); + } + } + @Override public void evict(CalculatedFieldId calculatedFieldId) { CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); @@ -243,7 +226,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { calculatedFieldLinks.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); - entityIdCalculatedFields.remove(oldCalculatedField.getEntityId()); log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldsCtx.remove(calculatedFieldId); log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0de3136439..dd11c799c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -92,6 +92,7 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -253,7 +254,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); + CalculatedField cf = calculatedFieldCache.getCalculatedField(calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -263,7 +264,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (cf != null) { EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -297,7 +298,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); @@ -345,17 +346,27 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (supportedReferencedEntities.contains(entityId.getEntityType())) { TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); - Map> tpiStatesToUpdate = new HashMap<>(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - updateTelemetryForEntity(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); - updateTelemetryForProfile(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId), tpiStatesToUpdate); - updateTelemetryForLinkedEntities(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + if (tpi.isMyPartition()) { - if (!tpiStatesToUpdate.isEmpty()) { - tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); - clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); - }); + processCalculatedFields(calculatedFieldTelemetryUpdateRequest, entityId); + processCalculatedFields(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId)); + + Map> tpiStatesToUpdate = new HashMap<>(); + processCalculatedFieldLinks(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() + .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + }); + } + } else { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() + .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + // Forward this request to a correct server based on entity id. } } } catch (Exception e) { @@ -363,30 +374,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { - updateTelemetryForEntity(request, request.getEntityId(), tpiStates); - } - - private void updateTelemetryForProfile(CalculatedFieldTelemetryUpdateRequest request, EntityId profileId, Map> tpiStates) { - updateTelemetryForEntity(request, profileId, tpiStates); - } - - private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, Map> tpiStates) { + private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - if (targetEntity != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(targetEntity); - mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); - }); - } - } else { - List ctxIds = tpiStates.computeIfAbsent(tpi, k -> new ArrayList<>()); - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { - ctxIds.add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); + if (cfTargetEntityId != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(cfTargetEntityId).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(cfTargetEntityId); + mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); }); } } @@ -405,14 +400,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForLinkedEntities(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId) + calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - EntityId targetEntityId = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId).getEntityId(); + EntityId targetEntityId = calculatedFieldCache.getCalculatedField(calculatedFieldId).getEntityId(); if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { @@ -451,33 +446,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); + + if (proto.getLinksList().isEmpty()) { + onTelemetryUpdate(request); + return; + } proto.getLinksList().forEach(ctxIdProto -> { - EntityId entityId = EntityIdFactory.getByTypeAndUuid( - ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - - List updatedTelemetry = proto.getUpdatedTelemetryList().stream() - .map(ProtoUtils::fromTelemetryProto) - .toList(); - - boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); - - CalculatedFieldTelemetryUpdateRequest request = attributesUpdated - ? new CalculatedFieldAttributeUpdateRequest( - tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()) - : new CalculatedFieldTimeSeriesUpdateRequest( - tenantId, entityId, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - onTelemetryUpdate(request); + CalculatedFieldLinkConfiguration linkConfiguration + = calculatedFieldCache.getCalculatedField(calculatedFieldId).getConfiguration().getReferencedEntityConfig(entityId); + + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, linkConfiguration); }); } catch (Exception e) { log.trace("Failed to process telemetry update msg: [{}]", proto, e); @@ -486,8 +470,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); @@ -524,7 +508,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map argumentsMap = proto.getArgumentsMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); @@ -559,7 +543,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - getCalculatedFieldLinks(tenantId, entityId, profileId) + getCalculatedFieldLinks(entityId, profileId) .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -585,7 +569,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) .stream() - .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(tenantId, cfId, tbelInvokeService)) + .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(cfId, tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -722,10 +706,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private List getCalculatedFieldLinks(TenantId tenantId, EntityId entityId, EntityId profileId) { - List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + private List getCalculatedFieldLinks(EntityId entityId, EntityId profileId) { + List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId)); if (profileId != null) { - links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId)); + links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(profileId)); } return links; } @@ -870,13 +854,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { + return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); + } + + ; + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( CalculatedFieldTelemetryUpdateRequest request, List links ) { TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()); + .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()) + .setEntityType(request.getEntityId().getEntityType().name()) + .setEntityIdMSB(request.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(request.getEntityId().getId().getLeastSignificantBits()); for (CalculatedFieldEntityCtxId link : links) { builder.addLinks(toProto(link)); @@ -904,6 +897,31 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return builder.build(); } + private CalculatedFieldTelemetryUpdateRequest fromProto(TransportProtos.TelemetryUpdateMsgProto proto) { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + + List updatedTelemetry = proto.getUpdatedTelemetryList().stream() + .map(ProtoUtils::fromTelemetryProto) + .toList(); + + boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); + + return attributesUpdated + ? new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()) + : new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + } + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 2aaed13ec1..dac35bfc5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -194,7 +194,9 @@ public abstract class AbstractConsumerService buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId) { + return getReferencedEntities().stream() + .filter(referencedEntity -> !referencedEntity.equals(cfEntityId)) + .map(referencedEntityId -> buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId) { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedFieldId); + link.setConfiguration(getReferencedEntityConfig(referencedEntityId)); + return link; + } + @Override public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { ObjectNode configNode = mapper.createObjectNode(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 5c428bd628..ac94ade134 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -20,9 +20,12 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.List; import java.util.Map; @@ -57,4 +60,8 @@ public interface CalculatedFieldConfiguration { @JsonIgnore JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); + List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); + + CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 685ca47719..8a7c2d8c03 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -824,10 +824,13 @@ message ProfileEntityMsgProto { message TelemetryUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - repeated CalculatedFieldEntityCtxIdProto links = 3; - repeated CalculatedFieldIdProto previousCalculatedFields = 4; - string scope = 5; - repeated TelemetryProto updatedTelemetry = 6; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + repeated CalculatedFieldEntityCtxIdProto links = 6; + repeated CalculatedFieldIdProto previousCalculatedFields = 7; + string scope = 8; + repeated TelemetryProto updatedTelemetry = 9; } message CalculatedFieldEntityCtxIdProto { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 36bc3d038a..9c81d91f64 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -38,7 +38,6 @@ import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @@ -240,23 +239,8 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - List links = buildCalculatedFieldLinks(tenantId, calculatedField); + List links = calculatedField.getConfiguration().buildCalculatedFieldLinks(tenantId, calculatedField.getEntityId(), calculatedField.getId()); links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); } - private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); - return cfConfig.getReferencedEntities().stream() - .filter(referencedEntity -> !referencedEntity.equals(calculatedField.getEntityId())) - .map(referencedEntityId -> { - CalculatedFieldLink link = new CalculatedFieldLink(); - link.setTenantId(tenantId); - link.setEntityId(referencedEntityId); - link.setCalculatedFieldId(calculatedField.getId()); - link.setConfiguration(cfConfig.getReferencedEntityConfig(referencedEntityId)); - return link; - }) - .collect(Collectors.toList()); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 816fa1546c..bed6f2d3a2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -29,7 +28,7 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); - List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); List findAllByTenantId(UUID tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 20081299e8..cdcffdd440 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -57,7 +57,7 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); } @Override From b5ca7e655fc33b5d1acc75a59e2306d62470ef93 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 9 Jan 2025 10:29:38 +0200 Subject: [PATCH 078/416] Increate log level to warn for evalSha --- .../server/cache/RedisTbTransactionalCache.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index 1eee03ccf9..c3ce4b356f 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -286,13 +286,13 @@ public abstract class RedisTbTransactionalCache Date: Thu, 9 Jan 2025 11:10:35 +0200 Subject: [PATCH 079/416] Improvement for logs in executeScript --- .../server/cache/RedisTbTransactionalCache.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index c3ce4b356f..97fc36fe27 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -286,13 +286,13 @@ public abstract class RedisTbTransactionalCache Date: Thu, 9 Jan 2025 14:45:42 +0200 Subject: [PATCH 080/416] added calculated field state service --- .../service/cf/CalculatedFieldCache.java | 2 - .../cf/CalculatedFieldExecutionService.java | 2 - .../cf/DefaultCalculatedFieldCache.java | 51 +-- ...efaultCalculatedFieldExecutionService.java | 325 ++++++------------ .../server/service/cf/RocksDBService.java | 13 - .../queue/DefaultTbCoreConsumerService.java | 14 - .../server/common/util/ProtoUtils.java | 40 --- common/proto/src/main/proto/queue.proto | 43 +-- 8 files changed, 119 insertions(+), 371 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 7394c95f08..bf5dc8d42f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -36,8 +36,6 @@ public interface CalculatedFieldCache { List getCalculatedFieldLinksByEntityId(EntityId entityId); - void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6d1d459b9b..8ba1f6dfed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -27,8 +27,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); - void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); - void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 71fc3eff67..f762ae3530 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -36,12 +36,12 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -72,14 +72,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); calculatedFields.values().forEach(cf -> - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cf) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf) ); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link)); calculatedFieldLinks.values().stream() .flatMap(List::stream) .forEach(link -> - entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link) + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) ); } @@ -90,41 +90,17 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldsByEntityId(EntityId entityId) { - return entityIdCalculatedFields.getOrDefault(entityId, new ArrayList<>()); + return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); } @Override public List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { - return calculatedFieldLinks.getOrDefault(calculatedFieldId, new ArrayList<>()); + return calculatedFieldLinks.getOrDefault(calculatedFieldId, new CopyOnWriteArrayList<>()); } @Override public List getCalculatedFieldLinksByEntityId(EntityId entityId) { - return entityIdCalculatedFieldLinks.getOrDefault(entityId, new ArrayList<>()); - } - - @Override - public void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { - log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); - calculatedFieldFetchLock.lock(); - try { - List cfLinks = getCalculatedFieldLinks(calculatedFieldId); - if (cfLinks != null && !cfLinks.isEmpty()) { - cfLinks.forEach(link -> { - entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { - if (existingList == null) { - existingList = new ArrayList<>(); - } else if (!(existingList instanceof ArrayList)) { - existingList = new ArrayList<>(existingList); - } - existingList.add(link); - return existingList; - }); - }); - } - } finally { - calculatedFieldFetchLock.unlock(); - } + return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); } @Override @@ -192,7 +168,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { calculatedFields.put(calculatedFieldId, calculatedField); - entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new ArrayList<>()).add(calculatedField); + entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); @@ -200,7 +176,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { configuration.getReferencedEntities().stream() .filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) .forEach(referencedEntityId -> { - entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new ArrayList<>()) + entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new CopyOnWriteArrayList<>()) .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); }); } finally { @@ -210,13 +186,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - calculatedFieldFetchLock.lock(); - try { - evict(calculatedFieldId); - addCalculatedField(tenantId, calculatedFieldId); - } finally { - calculatedFieldFetchLock.unlock(); - } + evict(calculatedFieldId); + addCalculatedField(tenantId, calculatedFieldId); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index dd11c799c2..e54c1bd5c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -77,6 +77,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -107,8 +108,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; -import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @@ -122,7 +121,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; + private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; @@ -148,8 +147,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(() -> rocksDBService.getAll() - .forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class)))); + scheduledExecutor.submit(() -> states.putAll(stateService.restoreStates())); } @PreDestroy @@ -223,10 +221,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + CalculatedFieldEntityCtx restoredCtx = stateService.restoreState(ctxId); - if (storedState != null) { - CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); + if (restoredCtx != null) { states.put(ctxId, restoredCtx); log.info("Restored state for CalculatedField [{}]", calculatedFieldId); } else { @@ -251,18 +248,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); + calculatedFieldCache.evict(calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldCache.getCalculatedField(calculatedFieldId); + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + calculatedFieldCache.updateCalculatedField(tenantId, calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); if (!shouldReinit) { return; } } if (cf != null) { + calculatedFieldCache.addCalculatedField(tenantId, calculatedFieldId); EntityId entityId = cf.getEntityId(); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { @@ -312,12 +312,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); - List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId)) - .map(JacksonUtil::writeValueAsString) - .toList(); - rocksDBService.deleteAll(statesToRemove); + states.keySet().removeIf(ctxId -> { + if (ctxId.cfId().equals(calculatedFieldId)) { + stateService.removeState(ctxId); + return true; + } + return false; + }); } catch (Exception e) { log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); @@ -366,7 +367,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); - // Forward this request to a correct server based on entity id. } } } catch (Exception e) { @@ -386,20 +386,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForLinkedEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); - if (targetEntityTpi.isMyPartition()) { - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); - } else { - List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); - } - } - private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); @@ -411,14 +397,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - updateTelemetryForLinkedEntity(request, entityByProfile, link, tpiStates); + processCalculatedFieldLink(request, entityByProfile, link, tpiStates); }); } else { - updateTelemetryForLinkedEntity(request, targetEntityId, link, tpiStates); + processCalculatedFieldLink(request, targetEntityId, link, tpiStates); } }); } + private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + if (targetEntityTpi.isMyPartition()) { + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + } else { + List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); + ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); + } + } + private void mapAndProcessUpdatedTelemetry(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, @@ -490,31 +490,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - @Override - public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received CalculatedFieldStateMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}], entityId=[{}]", tenantId, calculatedFieldId, entityId); - if (proto.getClear()) { - clearState(tenantId, calculatedFieldId, entityId); - return; - } - - List previousCalculatedFieldIds = proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .collect(Collectors.toCollection(ArrayList::new)); - Map argumentsMap = proto.getArgumentsMap().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); - - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); - } catch (Exception e) { - log.trace("Failed to process calculated field update state msg: [{}]", proto, e); - } - } - @Override public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { @@ -522,12 +497,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) - .forEach(cfId -> clearState(tenantId, cfId, entityId)); - initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); + initializeStateForEntityByProfile(entityId, newProfileId, callback); + } else { + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setEntityProfileUpdateMsg(proto).build(), null); + } } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -539,37 +517,39 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - if (proto.getDeleted()) { - log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - getCalculatedFieldLinks(entityId, profileId) - .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + if (proto.getDeleted()) { + log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + + calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); + calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).forEach(cf -> clearState(cf.getId(), entityId)); + } else { + log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntityByProfile(entityId, profileId, callback); + } } else { - log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); } + + } catch (Exception e) { log.trace("Failed to process profile entity msg: [{}]", proto, e); } } - private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); - } + private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { + log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); + states.remove(ctxId); + stateService.removeState(ctxId); } - private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) - .stream() - .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(cfId, tbelInvokeService)) + private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).stream() + .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -607,65 +587,58 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { - TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); - states.compute(entityCtxId, (ctxId, ctx) -> { - CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); + states.compute(entityCtxId, (ctxId, ctx) -> { + CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); - CompletableFuture updateFuture = new CompletableFuture<>(); + CompletableFuture updateFuture = new CompletableFuture<>(); - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentsMap)) { - calculatedFieldEntityCtx.setState(state); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); - } - log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentsMap)) { + calculatedFieldEntityCtx.setState(state); + stateService.persistState(entityCtxId, calculatedFieldEntityCtx); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); } - updateFuture.complete(null); - }; + log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); + } + updateFuture.complete(null); + }; - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); - if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + if (!allKeysPresent || requiresTsRollingUpdate) { + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); - } + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + } else { + performUpdateState.accept(state); + } - try { - updateFuture.join(); - } catch (Exception e) { - log.trace("Failed to update state for ctxId [{}].", ctxId, e); - throw new RuntimeException("Failed to update or initialize state.", e); - } + try { + updateFuture.join(); + } catch (Exception e) { + log.trace("Failed to update state for ctxId [{}].", ctxId, e); + throw new RuntimeException("Failed to update or initialize state.", e); + } - return calculatedFieldEntityCtx; - }); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, previousCalculatedFieldIds, argumentsMap); - } + return calculatedFieldEntityCtx; + }); } private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List previousCalculatedFieldIds) { @@ -706,14 +679,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private List getCalculatedFieldLinks(EntityId entityId, EntityId profileId) { - List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId)); - if (profileId != null) { - links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(profileId)); - } - return links; - } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { Map argumentValues = new HashMap<>(); List> futures = new ArrayList<>(); @@ -777,89 +742,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List previousCalculatedFieldIds, Map argumentValues) { - TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); - if (argumentValues != null) { - argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); - } - if (previousCalculatedFieldIds != null) { - previousCalculatedFieldIds.forEach(cfId -> msgBuilder.addPreviousCalculatedFields( - TransportProtos.CalculatedFieldIdProto.newBuilder() - .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) - .build() - )); - } - - log.info("Sending calculated field state msg from entityId [{}]", entityId); - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); - } - - private void sendClearCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TransportProtos.CalculatedFieldStateMsgProto msg = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId) - .setClear(true) - .build(); - - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msg).build(), null); - } - - private TransportProtos.CalculatedFieldStateMsgProto.Builder createBaseCalculatedFieldStateMsg( - TenantId tenantId, - CalculatedFieldId calculatedFieldId, - EntityId entityId - ) { - return TransportProtos.CalculatedFieldStateMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - } - - private TransportProtos.ArgumentEntryProto toArgumentEntryProto(ArgumentEntry argumentEntry) { - TransportProtos.ArgumentEntryProto.Builder argumentProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); - - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); - tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> - tsRollingProtoBuilder.putTsRecords(ts, toObjectProto(value)) - ); - argumentProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - argumentProtoBuilder.setSingleValue( - TransportProtos.SingleValueProto.newBuilder() - .setTs(singleValueArgumentEntry.getTs()) - .setValue(toObjectProto(singleValueArgumentEntry.getValue())) - .build() - ); - } - - return argumentProtoBuilder.build(); - } - - private ArgumentEntry fromArgumentEntryProto(TransportProtos.ArgumentEntryProto entryProto) { - if (entryProto.hasTsRecords()) { - TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); - entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> - tsRollingArgumentEntry.getTsRecords().put(ts, fromObjectProto(objectProto)) - ); - return tsRollingArgumentEntry; - } else if (entryProto.hasSingleValue()) { - TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); - return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue()), singleValueProto.getVersion()); - } else { - throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); - } - } - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); } - ; - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( CalculatedFieldTelemetryUpdateRequest request, List links ) { @@ -952,11 +838,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { + CalculatedFieldEntityCtx state = stateService.restoreState(entityCtxId); + + if (state == null) { return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + return state; } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index d6b2980042..3aed65eced 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; -import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; @@ -27,7 +26,6 @@ import org.thingsboard.server.utils.RocksDBConfig; import java.nio.charset.StandardCharsets; import java.util.HashMap; -import java.util.List; import java.util.Map; @Service @@ -59,17 +57,6 @@ public class RocksDBService { } } - public void deleteAll(List keys) { - try (WriteBatch batch = new WriteBatch()) { - for (String key : keys) { - batch.delete(key.getBytes(StandardCharsets.UTF_8)); - } - db.write(writeOptions, batch); - } catch (RocksDBException e) { - log.error("Failed to delete data from RocksDB", e); - } - } - public String get(String key) { try { byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 6baa75b3ef..c598540ff2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -326,8 +326,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldStateMsg(calculatedFieldStateMsgProto, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field state message for entityId [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index d332bac64f..073f47d59b 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -1316,46 +1316,6 @@ public class ProtoUtils { return builder.build(); } - public static TransportProtos.ObjectProto toObjectProto(Object value) { - if (value == null) { - throw new IllegalArgumentException("Cannot convert null to ObjectProto"); - } - - TransportProtos.ObjectProto.Builder builder = TransportProtos.ObjectProto.newBuilder(); - - if (value instanceof String) { - builder.setStringValue((String) value); - } else if (value instanceof Integer) { - builder.setIntValue((Integer) value); - } else if (value instanceof Long) { - builder.setLongValue((Long) value); - } else if (value instanceof Double) { - builder.setDoubleValue((Double) value); - } else if (value instanceof Boolean) { - builder.setBoolValue((Boolean) value); - } else { - throw new IllegalArgumentException("Unsupported value type: " + value.getClass().getName()); - } - - return builder.build(); - } - - public static Object fromObjectProto(TransportProtos.ObjectProto proto) { - try { - return switch (proto.getValueCase()) { - case STRINGVALUE -> proto.getStringValue(); - case INTVALUE -> proto.getIntValue(); - case LONGVALUE -> proto.getLongValue(); - case DOUBLEVALUE -> proto.getDoubleValue(); - case BOOLVALUE -> proto.getBoolValue(); - case VALUE_NOT_SET -> throw new IllegalArgumentException("Value not set in ObjectProto"); - }; - } catch (Exception e) { - log.error("Failed to deserialize ObjectProto: [{}]", proto, e); - return null; - } - } - private static boolean isNotNull(Object obj) { return obj != null; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 8a7c2d8c03..1036d5ba67 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -841,51 +841,11 @@ message CalculatedFieldEntityCtxIdProto { int64 entityIdLSB = 5; } -message CalculatedFieldStateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - string entityType = 5; - int64 entityIdMSB = 6; - int64 entityIdLSB = 7; - bool clear = 8; - repeated CalculatedFieldIdProto previousCalculatedFields = 9; - map arguments = 10; -} - message CalculatedFieldIdProto { int64 calculatedFieldIdMSB = 1; int64 calculatedFieldIdLSB = 2; } -message ArgumentEntryProto { - oneof entry_type { - TsRollingProto tsRecords = 1; - SingleValueProto singleValue = 2; - } -} - -message TsRollingProto { - map tsRecords = 1; -} - -message SingleValueProto { - int64 ts = 1; - ObjectProto value = 2; - int64 version = 3; -} - -message ObjectProto { - oneof value { - string stringValue = 1; - int32 intValue = 2; - int64 longValue = 3; - double doubleValue = 4; - bool boolValue = 5; - } -} - //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1632,7 +1592,6 @@ message ToCoreMsg { CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; ProfileEntityMsgProto profileEntityMsg = 55; - CalculatedFieldStateMsgProto calculatedFieldStateMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1681,6 +1640,8 @@ message ToRuleEngineMsg { repeated string relationTypes = 4; string failureMessage = 5; TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; + EntityProfileUpdateMsgProto entityProfileUpdateMsg = 7; + ProfileEntityMsgProto profileEntityMsg = 8; } message ToRuleEngineNotificationMsg { From 77e99d15df2c2a3f412c48fce87151d2a473be7f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 9 Jan 2025 14:51:17 +0200 Subject: [PATCH 081/416] added service files --- .../cf/ctx/CalculatedFieldStateService.java | 30 +++++++++ .../cf/ctx/state/RocksDBStateService.java | 64 +++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java new file mode 100644 index 0000000000..8bc5756f4e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx; + +import java.util.Map; + +public interface CalculatedFieldStateService { + + Map restoreStates(); + + CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId); + + void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state); + + void removeState(CalculatedFieldEntityCtxId ctxId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java new file mode 100644 index 0000000000..db8950804c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.service.cf.RocksDBService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; + +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${service.type:null}'=='monolith'") +public class RocksDBStateService implements CalculatedFieldStateService { + + private final RocksDBService rocksDBService; + + @Override + public Map restoreStates() { + return rocksDBService.getAll().entrySet().stream() + .collect(Collectors.toMap( + entry -> JacksonUtil.fromString(entry.getKey(), CalculatedFieldEntityCtxId.class), + entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldEntityCtx.class) + )); + } + + @Override + public CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId) { + return Optional.ofNullable(rocksDBService.get(JacksonUtil.writeValueAsString(ctxId))) + .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class)) + .orElse(null); + } + + @Override + public void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state) { + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(state)); + } + + @Override + public void removeState(CalculatedFieldEntityCtxId ctxId) { + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } + +} From 6611f017c7eae1efb05b7bcf0bdf09d17b13ad34 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 10 Jan 2025 16:58:26 +0200 Subject: [PATCH 082/416] changed Argument structure --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 8 ++ ...efaultCalculatedFieldExecutionService.java | 125 ++++++------------ .../cf/ctx/state/CalculatedFieldCtx.java | 15 ++- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- ...CalculatedFieldAttributeUpdateRequest.java | 25 ++++ ...CalculatedFieldTelemetryUpdateRequest.java | 3 + ...alculatedFieldTimeSeriesUpdateRequest.java | 31 +++++ .../TbRuleEngineQueueConsumerManager.java | 5 + .../CalculatedFieldControllerTest.java | 2 +- .../data/cf/configuration/Argument.java | 9 +- .../BaseCalculatedFieldConfiguration.java | 49 +++---- .../cf/configuration/ReferencedEntityKey.java | 30 +++++ .../dao/model/sql/CalculatedFieldEntity.java | 3 + .../server/dao/service/AssetServiceTest.java | 2 +- .../service/CalculatedFieldServiceTest.java | 2 +- .../dao/service/CustomerServiceTest.java | 2 +- .../server/dao/service/DeviceServiceTest.java | 2 +- 18 files changed, 195 insertions(+), 122 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index bf5dc8d42f..8730aeeedf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -38,6 +38,8 @@ public interface CalculatedFieldCache { CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService); + Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index f762ae3530..868001d0d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -61,6 +61,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldCtxs = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); @Value("${calculatedField.initFetchPackSize:50000}") @@ -126,6 +127,13 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return ctx; } + @Override + public List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService) { + return getCalculatedFieldsByEntityId(entityId).stream() + .map(cf -> getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) + .toList(); + } + @Override public Set getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) { Set entities = profileEntities.get(entityProfileId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index e54c1bd5c9..7c17d54649 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,8 +39,6 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -273,7 +271,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) + .filter(entry -> !isProfileEntity(entry.getValue().getRefEntityId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { @@ -341,30 +339,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { + public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest request) { try { - EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); + EntityId entityId = request.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { - TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + TenantId tenantId = request.getTenantId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - processCalculatedFields(calculatedFieldTelemetryUpdateRequest, entityId); - processCalculatedFields(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId)); + processCalculatedFields(request, entityId); + processCalculatedFields(request, getProfileId(tenantId, entityId)); Map> tpiStatesToUpdate = new HashMap<>(); - processCalculatedFieldLinks(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + processCalculatedFieldLinks(request, tpiStatesToUpdate); if (!tpiStatesToUpdate.isEmpty()) { tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request, ctxIds); clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); }); } } else { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request); clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); } @@ -375,13 +373,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - if (cfTargetEntityId != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(cfTargetEntityId).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(cfTargetEntityId); - mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); + calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } }); } } @@ -393,56 +390,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - EntityId targetEntityId = calculatedFieldCache.getCalculatedField(calculatedFieldId).getEntityId(); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + EntityId targetEntityId = ctx.getEntityId(); if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - processCalculatedFieldLink(request, entityByProfile, link, tpiStates); + processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); }); } else { - processCalculatedFieldLink(request, targetEntityId, link, tpiStates); + processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); } }); } - private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); if (targetEntityTpi.isMyPartition()) { - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); - } - } - - private void mapAndProcessUpdatedTelemetry(TenantId tenantId, - EntityId entityId, - CalculatedFieldId calculatedFieldId, - CalculatedFieldTelemetryUpdateRequest request, - CalculatedFieldLinkConfiguration linkConfiguration) { - Map telemetryKeys = request.getTelemetryKeysFromLink(linkConfiguration); - Map updatedTelemetry = mapTelemetryKeys(telemetryKeys, request.getKvEntries()); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); + ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getCfId(), targetEntity)); } } - private Map mapTelemetryKeys(Map telemetryKeys, List kvEntries) { - return kvEntries.stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> telemetryKeys.getOrDefault(entry.getKey(), entry.getKey()), - entry -> entry, - (v1, v2) -> v1 - )); - } - @Override public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { try { @@ -454,40 +427,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } proto.getLinksList().forEach(ctxIdProto -> { - TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - CalculatedFieldLinkConfiguration linkConfiguration - = calculatedFieldCache.getCalculatedField(calculatedFieldId).getConfiguration().getReferencedEntityConfig(entityId); - - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, linkConfiguration); + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, entityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } }); } catch (Exception e) { log.trace("Failed to process telemetry update msg: [{}]", proto, e); } } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { - log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { + log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - EntityId cfEntityId = calculatedField.getEntityId(); - switch (cfEntityId.getEntityType()) { - case ASSET_PROFILE, DEVICE_PROFILE -> { - boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); - if (isCommonEntity) { - calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, previousCalculatedFieldIds)); - } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, previousCalculatedFieldIds); - } - } - default -> - updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, previousCalculatedFieldIds); - } + updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); } @Override @@ -533,8 +492,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); } - - } catch (Exception e) { log.trace("Failed to process profile entity msg: [{}]", proto, e); } @@ -616,11 +573,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) @@ -696,7 +653,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getEntityId(); + EntityId argumentEntityId = argument.getRefEntityId(); EntityId entityId = isProfileEntity(argumentEntityId) ? targetEntityId : argumentEntityId; @@ -704,17 +661,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - return switch (argument.getType()) { + return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); case ATTRIBUTE -> transformSingleValueArgument( Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()), result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), calculatedFieldCallbackExecutor) ); case TS_LATEST -> transformSingleValueArgument( Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + timeseriesService.findLatest(tenantId, entityId, argument.getRefEntityKey().getKey()), result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), calculatedFieldCallbackExecutor)); }; @@ -736,7 +693,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); @@ -826,7 +783,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private KvEntry createDefaultKvEntry(Argument argument) { - String key = argument.getKey(); + String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); if (NumberUtils.isParsable(defaultValue)) { return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index d54a3220ed..e17a1a61f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -22,13 +22,16 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.util.TbPair; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Data public class CalculatedFieldCtx { @@ -38,7 +41,8 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; - private final List argKeys; + private final Map, String> referencedEntityKeys; + private final List argNames; private Output output; private String expression; private TbelInvokeService tbelInvokeService; @@ -51,7 +55,12 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); - this.argKeys = new ArrayList<>(arguments.keySet()); + this.referencedEntityKeys = arguments.entrySet().stream() + .collect(Collectors.toMap( + entry -> new TbPair<>(entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), + Map.Entry::getKey + )); + this.argNames = new ArrayList<>(arguments.keySet()); this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; @@ -69,7 +78,7 @@ public class CalculatedFieldCtx { tenantId, tbelInvokeService, expression, - argKeys.toArray(String[]::new) + argNames.toArray(String[]::new) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index de7c514786..0421055fef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -49,7 +49,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); } }); - Object[] args = ctx.getArgKeys().stream() + Object[] args = ctx.getArgNames().stream() .map(key -> arguments.get(key).getValue()) .toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index a83cc0fc25..6050370fd2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -20,11 +20,16 @@ import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,4 +60,24 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel }; } + @Override + public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + Map mappedKvEntries = new HashMap<>(); + Map, String> referencedKeys = ctx.getReferencedEntityKeys(); + + kvEntries.forEach(entry -> { + String key = entry.getKey(); + + ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + + String argName = referencedKeys.get(new TbPair<>(entityId, referencedEntityKey)); + + if (argName != null) { + mappedKvEntries.put(argName, entry); + } + }); + + return mappedKvEntries; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 29ee899ec9..f85117dc41 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.List; import java.util.Map; @@ -36,4 +37,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); + Map getMappedTelemetry(CalculatedFieldCtx ctx); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 507daf386e..646145a46e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -19,11 +19,16 @@ import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -48,4 +53,30 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe return linkConfiguration.getTimeSeries(); } + @Override + public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + Map mappedKvEntries = new HashMap<>(); + Map, String> referencedKeys = ctx.getReferencedEntityKeys(); + + kvEntries.forEach(entry -> { + String key = entry.getKey(); + + ReferencedEntityKey tsLatestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + String argTsLatestName = referencedKeys.get(new TbPair<>(entityId, tsLatestKey)); + + if (argTsLatestName != null) { + mappedKvEntries.put(argTsLatestName, entry); + } else { + ReferencedEntityKey tsRollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); + String argTsRollingName = referencedKeys.get(new TbPair<>(entityId, tsRollingKey)); + + if (argTsRollingName != null) { + mappedKvEntries.put(argTsRollingName, entry); + } + } + }); + + return mappedKvEntries; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 243a3adbf7..83ab90e3bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -179,6 +180,10 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager getReferencedEntities() { return arguments.values().stream() - .map(Argument::getEntityId) + .map(Argument::getRefEntityId) .filter(Objects::nonNull) .collect(Collectors.toList()); } @@ -69,24 +70,24 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.entrySet().stream() - .filter(entry -> entry.getValue().getEntityId().equals(entityId)) + .filter(entry -> entry.getValue().getRefEntityId().equals(entityId)) .forEach(entry -> { - Argument targetArgument = entry.getValue(); - String argumentKey = entry.getKey(); + ReferencedEntityKey refEntityKey = entry.getValue().getRefEntityKey(); + String argumentName = entry.getKey(); - switch (targetArgument.getType()) { + switch (refEntityKey.getType()) { case ATTRIBUTE -> { - switch (targetArgument.getScope()) { + switch (refEntityKey.getScope()) { case CLIENT_SCOPE -> - linkConfiguration.getClientAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getClientAttributes().put(refEntityKey.getKey(), argumentName); case SERVER_SCOPE -> - linkConfiguration.getServerAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getServerAttributes().put(refEntityKey.getKey(), argumentName); case SHARED_SCOPE -> - linkConfiguration.getSharedAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getSharedAttributes().put(refEntityKey.getKey(), argumentName); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getTimeSeries().put(refEntityKey.getKey(), argumentName); } }); @@ -118,7 +119,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel ObjectNode argumentsNode = configNode.putObject("arguments"); arguments.forEach((key, argument) -> { ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getEntityId(); + EntityId referencedEntityId = argument.getRefEntityId(); if (referencedEntityId != null) { argumentNode.put("entityType", referencedEntityId.getEntityType().name()); argumentNode.put("entityId", referencedEntityId.getId().toString()); @@ -126,9 +127,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("entityType", entityType.name()); argumentNode.put("entityId", entityId.toString()); } - argumentNode.put("key", argument.getKey()); - argumentNode.put("type", String.valueOf(argument.getType())); - argumentNode.put("scope", String.valueOf(argument.getScope())); +// argumentNode.put("key", argument.getKey()); +// argumentNode.put("type", String.valueOf(argument.getType())); +// argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); @@ -165,19 +166,19 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { String referencedEntityType = argumentNode.get("entityType").asText(); UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); } else { - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); } - argument.setKey(argumentNode.get("key").asText()); +// argument.setRefEntityKey(argumentNode.get("key").asText()); JsonNode type = argumentNode.get("type"); - if (type != null && !type.isNull() && !type.asText().equals("null")) { - argument.setType(ArgumentType.valueOf(type.asText())); - } - JsonNode scope = argumentNode.get("scope"); - if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { - argument.setScope(AttributeScope.valueOf(scope.asText())); - } +// if (type != null && !type.isNull() && !type.asText().equals("null")) { +// argument.setType(ArgumentType.valueOf(type.asText())); +// } +// JsonNode scope = argumentNode.get("scope"); +// if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { +// argument.setScope(AttributeScope.valueOf(scope.asText())); +// } if (argumentNode.hasNonNull("defaultValue")) { argument.setDefaultValue(argumentNode.get("defaultValue").asText()); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java new file mode 100644 index 0000000000..b49495d959 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +@AllArgsConstructor +public class ReferencedEntityKey { + + private String key; + private ArgumentType type; + private AttributeScope scope; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 6aaaf05836..d1352a26b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -22,6 +22,7 @@ import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -95,6 +96,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); +// this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { @@ -112,6 +114,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); +// calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 9a0b9222f0..f38722c9c1 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -887,7 +887,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 5a8f7a2383..a9a70d8bc8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -156,7 +156,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index d0ee833261..236b159ddb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -382,7 +382,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 5b060ae145..cca30e6fbc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1225,7 +1225,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); From 345e4239732c2c801878445cd65ee369b61160cc Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Sun, 12 Jan 2025 14:27:46 +0200 Subject: [PATCH 083/416] Save time series strategies: update time series node config in rule chain JSONs; add null-check for persistence settings --- .../rule_chains/edge_root_rule_chain.json | 8 ++++++-- .../device_profile/rule_chain_template.json | 8 ++++++-- .../tenant/rule_chains/root_rule_chain.json | 6 +++++- .../src/main/resources/root_rule_chain.json | 20 ++++++++++++++----- .../engine/telemetry/TbMsgTimeseriesNode.java | 3 +++ 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json index 9adeb4f49e..e7b4b93e98 100644 --- a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json +++ b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json @@ -34,7 +34,11 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "configuration": { - "defaultTTL": 0 + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } }, "externalId": null }, @@ -185,4 +189,4 @@ ], "ruleChainConnections": null } -} \ No newline at end of file +} diff --git a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json index bce88d62b0..37cc226a6e 100644 --- a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json +++ b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json @@ -20,7 +20,11 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "configuration": { - "defaultTTL": 0 + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } } }, { @@ -134,4 +138,4 @@ ], "ruleChainConnections": null } -} \ No newline at end of file +} diff --git a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json index ee38849c1b..7893a64fd2 100644 --- a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json @@ -19,7 +19,11 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "configuration": { - "defaultTTL": 0 + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } } }, { diff --git a/monitoring/src/main/resources/root_rule_chain.json b/monitoring/src/main/resources/root_rule_chain.json index ff44ebfe79..a48fd6bf35 100644 --- a/monitoring/src/main/resources/root_rule_chain.json +++ b/monitoring/src/main/resources/root_rule_chain.json @@ -23,7 +23,11 @@ "singletonMode": false, "configurationVersion": 0, "configuration": { - "defaultTTL": 0 + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } }, "externalId": null }, @@ -275,7 +279,11 @@ "singletonMode": false, "configurationVersion": 0, "configuration": { - "defaultTTL": 0 + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } }, "externalId": null }, @@ -310,8 +318,10 @@ "configurationVersion": 0, "configuration": { "defaultTTL": 180, - "skipLatestPersistence": null, - "useServerTs": null + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } }, "externalId": null } @@ -415,4 +425,4 @@ ], "ruleChainConnections": null } -} \ No newline at end of file +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index b3eea21ece..1a014c5a43 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -83,6 +83,9 @@ public class TbMsgTimeseriesNode implements TbNode { ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); persistenceSettings = config.getPersistenceSettings(); + if (persistenceSettings == null) { + throw new TbNodeException("Persistence settings cannot be null!", true); + } } void onTenantProfileUpdate(TenantProfile tenantProfile) { From f8cfd158e279107e784014790accd2f689eb0b5a Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Sun, 12 Jan 2025 14:35:21 +0200 Subject: [PATCH 084/416] Save time series strategies: update one more time series node config in rule chain JSON --- .../src/test/resources/MqttRuleNodeTestMetadata.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json b/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json index c2bb52514e..adcb618cbe 100644 --- a/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json +++ b/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json @@ -39,8 +39,10 @@ "configurationVersion": 0, "configuration": { "defaultTTL": 0, - "skipLatestPersistence": false, - "useServerTs": false + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } }, "externalId": null }, From 698a0c19ecc61d654f61d0a52ea9ec7cdf68314a Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Mon, 13 Jan 2025 12:16:12 +0200 Subject: [PATCH 085/416] Save time series strategies: add SQL upgrade script --- .../main/data/upgrade/basic/schema_update.sql | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 64bbbaca05..0ff57119f4 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -209,3 +209,74 @@ $$; UPDATE resource SET resource_sub_type = 'EXTENSION' WHERE resource_type = 'JS_MODULE' AND resource_sub_type IS NULL; -- UPDATE RESOURCE JS_MODULE SUB TYPE END + +-- UPDATE SAVE TIME SERIES NODES START + +DO $$ + BEGIN + -- Check if the rule_node table exists + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_name = 'rule_node' + ) THEN + + -- CREATE JSON validation function + CREATE OR REPLACE FUNCTION is_valid_jsonb(input text) + RETURNS boolean + LANGUAGE plpgsql + AS $func$ + DECLARE + dummy JSONB; + BEGIN + dummy := input::jsonb; + RETURN true; + EXCEPTION + WHEN others THEN + RETURN false; + END; + $func$; + + UPDATE rule_node + SET configuration = CASE + -- Case 1: If configuration is NULL, invalid JSON, or not a JSON object - set default configuration + WHEN configuration IS NULL + OR NOT is_valid_jsonb(configuration) + OR jsonb_typeof(configuration::jsonb) <> 'object' + THEN jsonb_build_object( + 'defaultTTL', 0, + 'useServerTs', false, + 'persistenceSettings', jsonb_build_object('type', 'ON_EVERY_MESSAGE') + ) + -- Case 2: If a valid JSON object with persistenceSettings (rule node was already upgraded) - leave unchanged + WHEN configuration::jsonb ? 'persistenceSettings' + THEN configuration::jsonb + -- Case 3: If a valid JSON object without persistenceSettings and skipLatestPersistence = 'true' (string 'true' or boolean true) - set latest to SKIP + WHEN configuration::jsonb ->> 'skipLatestPersistence' = 'true' + THEN (configuration::jsonb - 'skipLatestPersistence') + || jsonb_build_object( + 'persistenceSettings', jsonb_build_object( + 'type', 'ADVANCED', + 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), + 'latest', jsonb_build_object('type', 'SKIP'), + 'webSockets', jsonb_build_object('type', 'ON_EVERY_MESSAGE') + ) + ) + -- Case 4: If a valid JSON object without persistenceSettings and skipLatestPersistence not 'true' (everything else) - set all to ON_EVERY_MESSAGE + ELSE (configuration::jsonb - 'skipLatestPersistence') + || jsonb_build_object( + 'persistenceSettings', jsonb_build_object( + 'type', 'ON_EVERY_MESSAGE' + ) + ) + END::text + WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode'; + + -- Drop the helper function + DROP FUNCTION is_valid_jsonb(text); + + END IF; + END; +$$; + +-- UPDATE SAVE TIME SERIES NODES END From 3908105ac5e4cba45100a2020beb61db1a1e34c2 Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 13 Jan 2025 13:23:41 +0200 Subject: [PATCH 086/416] fix_bug: tbel, in the isDecimal method. The mantissa format includes numbers written in exponential form --- .../java/org/thingsboard/script/api/tbel/TbUtils.java | 2 +- .../org/thingsboard/script/api/tbel/TbUtilsTest.java | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index fae91d14ca..ab1aef979a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1299,7 +1299,7 @@ public class TbUtils { if (str == null || str.isEmpty()) { return -1; } - return str.matches("[+-]?\\d+(\\.\\d+)?") ? DEC_RADIX : -1; + return str.matches("[+-]?\\d+(\\.\\d+)?([eE][+-]?\\d+)?") ? DEC_RADIX : -1; } public static int isHexadecimal(String str) { diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index 21ee8538a0..4ac387b258 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -442,8 +442,9 @@ public class TbUtilsTest { @Test public void parsDouble() { - String doubleValStr = "1729.1729"; - Assertions.assertEquals(java.util.Optional.of(doubleVal).get(), TbUtils.parseDouble(doubleValStr)); + String doubleValStr = "1.1428250947E8"; + Assertions.assertEquals(Double.parseDouble(doubleValStr), TbUtils.parseDouble(doubleValStr)); + doubleValStr = "1729.1729"; Assertions.assertEquals(0, Double.compare(doubleVal, TbUtils.parseHexToDouble(longValHex))); Assertions.assertEquals(0, Double.compare(doubleValRev, TbUtils.parseHexToDouble(longValHex, false))); Assertions.assertEquals(0, Double.compare(doubleVal, TbUtils.parseBigEndianHexToDouble(longValHex))); @@ -930,7 +931,13 @@ public class TbUtilsTest { @Test public void isDecimal_Test() { Assertions.assertEquals(10, TbUtils.isDecimal("4567039")); + Assertions.assertEquals(10, TbUtils.isDecimal("1.1428250947E8")); + Assertions.assertEquals(10, TbUtils.isDecimal("123.45")); + Assertions.assertEquals(10, TbUtils.isDecimal("-1.23E-4")); + Assertions.assertEquals(10, TbUtils.isDecimal("1E5")); Assertions.assertEquals(-1, TbUtils.isDecimal("C100110")); + Assertions.assertEquals(-1, TbUtils.isDecimal("abc")); + Assertions.assertEquals(-1, TbUtils.isDecimal(null)); } @Test From 631d314fcebe4f63c36597ead46549596bc1d2a4 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Mon, 13 Jan 2025 15:54:11 +0200 Subject: [PATCH 087/416] Save time series strategies: add Java upgrade script --- .../main/data/upgrade/basic/schema_update.sql | 5 +- .../engine/telemetry/TbMsgTimeseriesNode.java | 36 +++++++- .../telemetry/TbMsgTimeseriesNodeTest.java | 86 ++++++++++++++++++- 3 files changed, 121 insertions(+), 6 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 0ff57119f4..6aaf8c3dbd 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -269,8 +269,9 @@ DO $$ 'type', 'ON_EVERY_MESSAGE' ) ) - END::text - WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode'; + END::text, + configuration_version = 1 + WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' AND configuration_version = 0; -- Drop the helper function DROP FUNCTION is_valid_jsonb(text); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 1a014c5a43..db2421e307 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -15,8 +15,11 @@ */ package org.thingsboard.rule.engine.telemetry; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; @@ -24,6 +27,7 @@ import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.rule.engine.telemetry.strategy.PersistenceStrategy; import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TenantProfile; @@ -32,6 +36,7 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.TbMsg; import java.util.ArrayList; @@ -66,7 +71,8 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE "So, to make sure that all the messages will be processed correctly, one should enable this parameter for sequential message processing scenarios.", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbActionNodeTimeseriesConfig", - icon = "file_upload" + icon = "file_upload", + version = 1 ) public class TbMsgTimeseriesNode implements TbNode { @@ -182,4 +188,32 @@ public class TbMsgTimeseriesNode implements TbNode { ctx.removeListeners(); } + @Override + public TbPair upgrade(int fromVersion, JsonNode oldConfiguration) throws TbNodeException { + boolean hasChanges = false; + switch (fromVersion) { + case 0: + if (oldConfiguration.has("persistenceSettings")) { + break; + } + hasChanges = true; + JsonNode skipLatestPersistence = oldConfiguration.get("skipLatestPersistence"); + if (skipLatestPersistence != null && "true".equals(skipLatestPersistence.asText())) { + var skipLatestPersistenceSettings = new Advanced( + PersistenceStrategy.onEveryMessage(), + PersistenceStrategy.skip(), + PersistenceStrategy.onEveryMessage() + ); + ((ObjectNode) oldConfiguration).set("persistenceSettings", JacksonUtil.valueToTree(skipLatestPersistenceSettings)); + } else { + ((ObjectNode) oldConfiguration).set("persistenceSettings", JacksonUtil.valueToTree(new OnEveryMessage())); + } + ((ObjectNode) oldConfiguration).remove("skipLatestPersistence"); + break; + default: + break; + } + return new TbPair<>(hasChanges, oldConfiguration); + } + } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index ba8c614944..71d060aab2 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -25,11 +25,12 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.ThrowingConsumer; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; @@ -60,12 +61,13 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.assertArg; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) -public class TbMsgTimeseriesNodeTest { +public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("c8f34868-603a-4433-876a-7d356e5cf377")); private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("e5095e9a-04f4-44c9-b443-1cf1b97d3384")); @@ -82,7 +84,7 @@ public class TbMsgTimeseriesNodeTest { @BeforeEach public void setUp() throws TbNodeException { - node = new TbMsgTimeseriesNode(); + node = spy(new TbMsgTimeseriesNode()); config = new TbMsgTimeseriesNodeConfiguration().defaultConfiguration(); } @@ -291,4 +293,82 @@ public class TbMsgTimeseriesNodeTest { return expectedList; } + @Override + protected TbNode getTestNode() { + return node; + } + + private static Stream givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig() { + return Stream.of( + Arguments.of(0, """ + { + "defaultTTL": 0, + "useServerTs": false, + "skipLatestPersistence": false + }""", + true, + """ + { + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } + }"""), + Arguments.of(0, """ + { + "defaultTTL": 0, + "useServerTs": false + }""", + true, + """ + { + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } + }"""), + Arguments.of(0, """ + { + "defaultTTL": 0, + "useServerTs": false, + "skipLatestPersistence": null + }""", + true, + """ + { + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } + }"""), + Arguments.of(0, """ + { + "defaultTTL": 0, + "useServerTs": false, + "skipLatestPersistence": true + }""", + true, + """ + { + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ADVANCED", + "timeseries": { + "type": "ON_EVERY_MESSAGE" + }, + "latest": { + "type": "SKIP" + }, + "webSockets": { + "type": "ON_EVERY_MESSAGE" + } + } + }""") + ); + } + } From d1acba40a29391d59df1b5e96359664be1b503ea Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 14 Jan 2025 15:20:16 +0200 Subject: [PATCH 088/416] Save time series strategies: add tests for changes in telemetry service --- .../DefaultTbEntityViewService.java | 1 + .../DefaultTelemetrySubscriptionService.java | 7 +- ...faultTelemetrySubscriptionServiceTest.java | 387 ++++++++++++++++++ .../engine/api/TimeseriesSaveRequestTest.java | 33 ++ 4 files changed, 425 insertions(+), 3 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java create mode 100644 rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java index 8e0a901eed..ee807da750 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java @@ -350,6 +350,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen .entries(latestValues) .saveTimeseries(false) .saveLatest(true) + .sendWsUpdate(true) .callback(new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d7ff5fa5b7..dbd7967989 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -149,8 +149,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSendWsUpdate()) { addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); } - if (request.isSaveTimeseries() && request.isSaveLatest()) { - addEntityViewCallback(tenantId, entityId, request.getEntries()); + if (request.isSaveLatest()) { + copyLatestToEntityViews(tenantId, entityId, request.getEntries()); } return saveFuture; } @@ -205,7 +205,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { + private void copyLatestToEntityViews(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), new FutureCallback<>() { @@ -238,6 +238,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .entries(entityViewLatest) .saveTimeseries(false) .saveLatest(true) + .sendWsUpdate(true) .callback(new FutureCallback<>() { @Override public void onSuccess(@Nullable Void tmp) {} diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java new file mode 100644 index 0000000000..8162ef4555 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -0,0 +1,387 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.objects.AttributesEntityView; +import org.thingsboard.server.common.data.objects.TelemetryEntityView; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.stats.TbApiUsageReportClient; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.stream.LongStream; +import java.util.stream.Stream; + +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.lenient; + +@ExtendWith(MockitoExtension.class) +class DefaultTelemetrySubscriptionServiceTest { + + final TenantId tenantId = TenantId.fromUUID(UUID.fromString("a00ec470-c6b4-11ef-8c88-63b5533fb5bc")); + final CustomerId customerId = new CustomerId(UUID.fromString("7bdc9750-c775-11ef-8e03-ff69ed8da327")); + final EntityId entityId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + + final long sampleTtl = 10_000L; + + final List sampleTelemetry = List.of( + new BasicTsKvEntry(100L, new DoubleDataEntry("temperature", 65.2)), + new BasicTsKvEntry(100L, new DoubleDataEntry("humidity", 33.1)) + ); + + ApiUsageState apiUsageState; + + final TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .tenantId(tenantId) + .myPartition(true) + .build(); + + final FutureCallback emptyCallback = new FutureCallback<>() { + @Override + public void onSuccess(Void result) {} + + @Override + public void onFailure(@NonNull Throwable t) {} + }; + + ExecutorService wsCallBackExecutor; + ExecutorService tsCallBackExecutor; + + @Mock + TbClusterService clusterService; + @Mock + PartitionService partitionService; + @Mock + SubscriptionManagerService subscriptionManagerService; + @Mock + AttributesService attrService; + @Mock + TimeseriesService tsService; + @Mock + TbEntityViewService tbEntityViewService; + @Mock + TbApiUsageReportClient apiUsageClient; + @Mock + TbApiUsageStateService apiUsageStateService; + + DefaultTelemetrySubscriptionService telemetryService; + + @BeforeEach + void setup() { + telemetryService = new DefaultTelemetrySubscriptionService(attrService, tsService, tbEntityViewService, apiUsageClient, apiUsageStateService); + ReflectionTestUtils.setField(telemetryService, "clusterService", clusterService); + ReflectionTestUtils.setField(telemetryService, "partitionService", partitionService); + ReflectionTestUtils.setField(telemetryService, "subscriptionManagerService", Optional.of(subscriptionManagerService)); + + wsCallBackExecutor = MoreExecutors.newDirectExecutorService(); + ReflectionTestUtils.setField(telemetryService, "wsCallBackExecutor", wsCallBackExecutor); + + tsCallBackExecutor = MoreExecutors.newDirectExecutorService(); + ReflectionTestUtils.setField(telemetryService, "tsCallBackExecutor", tsCallBackExecutor); + + apiUsageState = new ApiUsageState(); + apiUsageState.setDbStorageState(ApiUsageStateValue.ENABLED); + lenient().when(apiUsageStateService.getApiUsageState(tenantId)).thenReturn(apiUsageState); + + lenient().when(partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId)).thenReturn(tpi); + + lenient().when(tsService.save(tenantId, entityId, sampleTelemetry, sampleTtl)).thenReturn(immediateFuture(sampleTelemetry.size())); + lenient().when(tsService.saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl)).thenReturn(immediateFuture(sampleTelemetry.size())); + lenient().when(tsService.saveLatest(tenantId, entityId, sampleTelemetry)).thenReturn(immediateFuture(listOfNNumbers(sampleTelemetry.size()))); + + // mock no entity views + lenient().when(tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId)).thenReturn(immediateFuture(Collections.emptyList())); + + // send partition change event so currentPartitions set is populated + telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)))); + } + + @AfterEach + void cleanup() { + wsCallBackExecutor.shutdownNow(); + tsCallBackExecutor.shutdownNow(); + } + + @Test + void shouldReportStorageDataPointsApiUsageWhenTimeSeriesIsSaved() { + // GIVEN + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(true) + .saveLatest(false) + .sendWsUpdate(false) + .callback(emptyCallback) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + then(apiUsageClient).should().report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, sampleTelemetry.size()); + } + + @Test + void shouldNotReportStorageDataPointsApiUsageWhenTimeSeriesIsNotSaved() { + // GIVEN + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(false) + .saveLatest(true) + .sendWsUpdate(true) + .callback(emptyCallback) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + then(apiUsageClient).shouldHaveNoInteractions(); + } + + @Test + void shouldThrowStorageDisabledWhenTimeSeriesIsSavedAndStorageIsDisabled() { + // GIVEN + apiUsageState.setDbStorageState(ApiUsageStateValue.DISABLED); + + SettableFuture future = SettableFuture.create(); + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(true) + .saveLatest(true) + .sendWsUpdate(true) + .future(future) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + assertThat(future).failsWithin(Duration.ofSeconds(5)) + .withThrowableOfType(ExecutionException.class) + .withCauseInstanceOf(RuntimeException.class) + .withMessageContaining("DB storage writes are disabled due to API limits!"); + } + + @Test + void shouldNotThrowStorageDisabledWhenTimeSeriesIsNotSavedAndStorageIsDisabled() { + // GIVEN + apiUsageState.setDbStorageState(ApiUsageStateValue.DISABLED); + + SettableFuture future = SettableFuture.create(); + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(false) + .saveLatest(true) + .sendWsUpdate(true) + .future(future) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + assertThat(future).succeedsWithin(Duration.ofSeconds(5)); + } + + @Test + void shouldCopyLatestToEntityViewWhenLatestIsSavedOnMainEntity() { + // GIVEN + var entityView = new EntityView(new EntityViewId(UUID.randomUUID())); + entityView.setTenantId(tenantId); + entityView.setCustomerId(customerId); + entityView.setEntityId(entityId); + entityView.setKeys(new TelemetryEntityView(sampleTelemetry.stream().map(KvEntry::getKey).toList(), new AttributesEntityView())); + + // mock that there is one entity view + given(tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId)).willReturn(immediateFuture(List.of(entityView))); + // mock that save latest call for entity view is successful + given(tsService.saveLatest(tenantId, entityView.getId(), sampleTelemetry)).willReturn(immediateFuture(listOfNNumbers(sampleTelemetry.size()))); + // mock TPI for entity view + given(partitionService.resolve(ServiceType.TB_CORE, tenantId, entityView.getId())).willReturn(tpi); + + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(false) + .saveLatest(true) + .sendWsUpdate(false) + .callback(emptyCallback) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + // should save latest to both the main entity and it's entity view + then(tsService).should().saveLatest(tenantId, entityId, sampleTelemetry); + then(tsService).should().saveLatest(tenantId, entityView.getId(), sampleTelemetry); + then(tsService).shouldHaveNoMoreInteractions(); + + // should send WS update only for entity view (WS update for the main entity is disabled in the save request) + then(subscriptionManagerService).should().onTimeSeriesUpdate(tenantId, entityView.getId(), sampleTelemetry, TbCallback.EMPTY); + then(subscriptionManagerService).shouldHaveNoMoreInteractions(); + } + + @Test + void shouldNotCopyLatestToEntityViewWhenLatestIsNotSavedOnMainEntity() { + // GIVEN + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(true) + .saveLatest(false) + .sendWsUpdate(false) + .callback(emptyCallback) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + // should save only time series for the main entity + then(tsService).should().saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl); + then(tsService).shouldHaveNoMoreInteractions(); + + // should not send any WS updates + then(subscriptionManagerService).shouldHaveNoInteractions(); + } + + @ParameterizedTest + @MethodSource("booleanCombinations") + void shouldCallCorrectApiBasedOnBooleanFlagsInTheSaveRequest(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { + // GIVEN + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(saveTimeseries) + .saveLatest(saveLatest) + .sendWsUpdate(sendWsUpdate) + .callback(emptyCallback) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + if (saveTimeseries && saveLatest) { + then(tsService).should().save(tenantId, entityId, sampleTelemetry, sampleTtl); + } else if (saveLatest) { + then(tsService).should().saveLatest(tenantId, entityId, sampleTelemetry); + } else if (saveTimeseries) { + then(tsService).should().saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl); + } + then(tsService).shouldHaveNoMoreInteractions(); + + if (sendWsUpdate) { + then(subscriptionManagerService).should().onTimeSeriesUpdate(tenantId, entityId, sampleTelemetry, TbCallback.EMPTY); + } else { + then(subscriptionManagerService).shouldHaveNoInteractions(); + } + } + + private static Stream booleanCombinations() { + return Stream.of( + Arguments.of(true, true, true), + Arguments.of(true, true, false), + Arguments.of(true, false, true), + Arguments.of(true, false, false), + Arguments.of(false, true, true), + Arguments.of(false, true, false), + Arguments.of(false, false, true), + Arguments.of(false, false, false) + ); + } + + // used to emulate sequence numbers returned by save latest API + private static List listOfNNumbers(int N) { + return LongStream.range(0, N).boxed().toList(); + } + +} diff --git a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java new file mode 100644 index 0000000000..cf05f9905b --- /dev/null +++ b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class TimeseriesSaveRequestTest { + + @Test + void testBooleanFlagsDefaultToTrue() { + var request = TimeseriesSaveRequest.builder().build(); + + assertThat(request.isSaveTimeseries()).isTrue(); + assertThat(request.isSaveLatest()).isTrue(); + assertThat(request.isSendWsUpdate()).isTrue(); + } + +} From 4eedea076633f4e1a6aef0245564bdf1510267ae Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 15 Jan 2025 16:11:54 +0200 Subject: [PATCH 089/416] EntityDataProto: add ts to correctly handle PostAttributeMsg update from/to Edge --- .../telemetry/EntityDataMsgConstructor.java | 6 +++ .../telemetry/BaseTelemetryProcessor.java | 46 ++++++++++++---- .../server/edge/DeviceEdgeTest.java | 53 +++++++++++++++++++ common/edge-api/src/main/proto/edge.proto | 1 + .../server/common/adaptor/JsonConverter.java | 10 ++-- .../common/transport/util/JsonUtils.java | 4 +- .../engine/edge/AbstractTbMsgPushNode.java | 1 + 7 files changed, 107 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/telemetry/EntityDataMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/telemetry/EntityDataMsgConstructor.java index 6493f97f57..f7f3cc31dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/telemetry/EntityDataMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/telemetry/EntityDataMsgConstructor.java @@ -67,6 +67,9 @@ public class EntityDataMsgConstructor { TransportProtos.PostAttributeMsg attributesUpdatedMsg = JsonConverter.convertToAttributesProto(data.getAsJsonObject("kv")); builder.setAttributesUpdatedMsg(attributesUpdatedMsg); builder.setPostAttributeScope(getScopeOfDefault(data)); + if (data.get("ts") != null && !data.get("ts").isJsonNull()) { + builder.setAttributeTs(data.getAsJsonPrimitive("ts").getAsLong()); + } } catch (Exception e) { log.warn("[{}][{}] Can't convert to AttributesUpdatedMsg proto, entityData [{}]", tenantId, entityId, entityData, e); } @@ -77,6 +80,9 @@ public class EntityDataMsgConstructor { TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(data.getAsJsonObject("kv")); builder.setPostAttributesMsg(postAttributesMsg); builder.setPostAttributeScope(getScopeOfDefault(data)); + if (data.get("ts") != null && !data.get("ts").isJsonNull()) { + builder.setAttributeTs(data.getAsJsonPrimitive("ts").getAsLong()); + } } catch (Exception e) { log.warn("[{}][{}] Can't convert to PostAttributesMsg, entityData [{}]", tenantId, entityId, entityData, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java index 488fc3d8b9..520c7ef5ff 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java @@ -54,6 +54,7 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -79,7 +80,9 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { @@ -114,7 +117,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { abstract protected String getMsgSourceKey(); - public List> processTelemetryMsg(TenantId tenantId, EntityDataProto entityData) { + public List> processTelemetryMsg(TenantId tenantId, EntityDataProto entityData) throws Exception { log.trace("[{}] processTelemetryMsg [{}]", tenantId, entityData); List> result = new ArrayList<>(); EntityId entityId = constructEntityId(entityData.getEntityType(), entityData.getEntityIdMSB(), entityData.getEntityIdLSB()); @@ -125,11 +128,13 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { CustomerId customerId = pair.getValue(); metaData.putValue(DataConstants.MSG_SOURCE_KEY, getMsgSourceKey()); if (entityData.hasPostAttributesMsg()) { - result.add(processPostAttributes(tenantId, customerId, entityId, entityData.getPostAttributesMsg(), metaData)); + long ts = entityData.hasAttributeTs() ? entityData.getAttributeTs() : System.currentTimeMillis(); + result.add(processPostAttributes(tenantId, customerId, entityId, entityData.getPostAttributesMsg(), metaData, ts)); } if (entityData.hasAttributesUpdatedMsg()) { metaData.putValue("scope", entityData.getPostAttributeScope()); - result.add(processAttributesUpdate(tenantId, customerId, entityId, entityData.getAttributesUpdatedMsg(), metaData)); + long ts = entityData.hasAttributeTs() ? entityData.getAttributeTs() : System.currentTimeMillis(); + result.add(processAttributesUpdate(tenantId, customerId, entityId, entityData.getAttributesUpdatedMsg(), metaData, ts)); } if (entityData.hasPostTelemetryMsg()) { result.add(processPostTelemetry(tenantId, customerId, entityId, entityData.getPostTelemetryMsg(), metaData)); @@ -257,9 +262,13 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { return new ImmutablePair<>(queueName, ruleChainId); } - private ListenableFuture processPostAttributes(TenantId tenantId, CustomerId customerId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData) { + private ListenableFuture processPostAttributes(TenantId tenantId, CustomerId customerId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData, long ts) throws Exception { SettableFuture futureToSet = SettableFuture.create(); JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); + + List attributes = new ArrayList<>(JsonConverter.convertToAttributes(json, ts)); + filterAttributesByTs(tenantId, entityId, AttributeScope.CLIENT_SCOPE, attributes, json); + var defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId); TbMsg tbMsg = TbMsg.newMsg() .queueName(defaultQueueAndRuleChain.getKey()) @@ -289,16 +298,18 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { CustomerId customerId, EntityId entityId, TransportProtos.PostAttributeMsg msg, - TbMsgMetaData metaData) { + TbMsgMetaData metaData, + long ts) throws Exception { SettableFuture futureToSet = SettableFuture.create(); JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); - List attributes = new ArrayList<>(JsonConverter.convertToAttributes(json)); - String scope = metaData.getValue("scope"); + AttributeScope scope = AttributeScope.valueOf(metaData.getValue("scope")); + List attributes = new ArrayList<>(JsonConverter.convertToAttributes(json, ts)); + List attributesToSave = filterAttributesByTs(tenantId, entityId, scope, attributes, json); tsSubService.saveAttributes(AttributesSaveRequest.builder() .tenantId(tenantId) .entityId(entityId) - .scope(AttributeScope.valueOf(scope)) - .entries(attributes) + .scope(scope) + .entries(attributesToSave) .callback(new FutureCallback<>() { @Override public void onSuccess(@Nullable Void tmp) { @@ -390,4 +401,21 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { entityDataMsgConstructor.constructEntityDataMsg(tenantId, entityId, actionType, JsonParser.parseString(bodyJackson)); } + private List filterAttributesByTs(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes, JsonObject jsonObject) throws Exception { + List keys = attributes.stream().map(KvEntry::getKey).toList(); + Map existingAttributesTs = edgeCtx.getAttributesService().find(tenantId, entityId, scope, keys).get() + .stream().collect(Collectors.toMap(KvEntry::getKey, AttributeKvEntry::getLastUpdateTs)); + return attributes.stream() + .filter(attribute -> { + String key = attribute.getKey(); + long incomingTs = attribute.getLastUpdateTs(); + if (incomingTs > existingAttributesTs.getOrDefault(key, 0L)) { + return true; + } else { + jsonObject.remove(key); + return false; + } + }).toList(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java index 387011717a..11bc053bb5 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java @@ -488,6 +488,59 @@ public class DeviceEdgeTest extends AbstractEdgeTest { doDelete("/api/plugins/telemetry/DEVICE/" + device.getId().getId() + "/SERVER_SCOPE?keys=" + attributesKey, String.class); } + @Test + public void testSendOutdatedAttributeToCloud() throws Exception { + long ts = System.currentTimeMillis(); + Device device = saveDeviceOnCloudAndVerifyDeliveryToEdge(); + + edgeImitator.expectResponsesAmount(1); + + ObjectNode attributesNode = JacksonUtil.newObjectNode(); + String originalValue = "original_value"; + attributesNode.put("test_attr", originalValue); + doPost("/api/plugins/telemetry/DEVICE/" + device.getId() + "/attributes/SERVER_SCOPE", attributesNode); + + JsonObject attributesData = new JsonObject(); + // incorrect msg, will not be saved, because of ts is lower than for already existing + String attributesKey = "test_attr"; + String attributeValueIncorrect = "test_value"; + // correct msg, will be saved, no ts issue + String attributeKey2 = "test_attr2"; + String attributeValue2Correct = "test_value2"; + attributesData.addProperty(attributesKey, attributeValueIncorrect); + attributesData.addProperty(attributeKey2, attributeValue2Correct); + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); + EntityDataProto.Builder entityDataBuilder = EntityDataProto.newBuilder(); + entityDataBuilder.setEntityType(device.getId().getEntityType().name()); + entityDataBuilder.setEntityIdMSB(device.getId().getId().getMostSignificantBits()); + entityDataBuilder.setEntityIdLSB(device.getId().getId().getLeastSignificantBits()); + entityDataBuilder.setAttributesUpdatedMsg(JsonConverter.convertToAttributesProto(attributesData)); + entityDataBuilder.setPostAttributeScope(DataConstants.SERVER_SCOPE); + entityDataBuilder.setAttributeTs(ts); + + uplinkMsgBuilder.addEntityData(entityDataBuilder.build()); + + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + Assert.assertTrue(edgeImitator.waitForResponses()); + + String attributeValuesUrl = "/api/plugins/telemetry/DEVICE/" + device.getId() + "/values/attributes/" + DataConstants.SERVER_SCOPE; + List> attributes = doGetAsyncTyped(attributeValuesUrl, new TypeReference<>() { + }); + + Optional> customAttributeOpt = getAttributeByKey(attributesKey, attributes); + Assert.assertTrue(customAttributeOpt.isPresent()); + Map customAttribute = customAttributeOpt.get(); + Assert.assertNotEquals(attributeValueIncorrect, customAttribute.get("value")); + Assert.assertEquals(originalValue, customAttribute.get("value")); + + customAttributeOpt = getAttributeByKey(attributeKey2, attributes); + Assert.assertTrue(customAttributeOpt.isPresent()); + customAttribute = customAttributeOpt.get(); + Assert.assertEquals(attributeValue2Correct, customAttribute.get("value")); + + doDelete("/api/plugins/telemetry/DEVICE/" + device.getId().getId() + "/SERVER_SCOPE?keys=" + attributesKey, String.class); + } + @Test public void testSendDeviceToCloudWithNameThatAlreadyExistsOnCloud() throws Exception { String deviceOnCloudName = StringUtils.randomAlphanumeric(15); diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 4dc762e14c..0557c5f5d4 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -132,6 +132,7 @@ message EntityDataProto { transport.PostAttributeMsg attributesUpdatedMsg = 6; string postAttributeScope = 7; AttributeDeleteMsg attributeDeleteMsg = 8; + optional int64 attributeTs = 9; } message AttributeDeleteMsg { diff --git a/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java index f4abf5f3a0..9226395289 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java @@ -63,7 +63,6 @@ import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.function.Consumer; -import java.util.stream.Collectors; public class JsonConverter { @@ -540,10 +539,12 @@ public class JsonConverter { } public static Set convertToAttributes(JsonElement element) { - Set result = new HashSet<>(); long ts = System.currentTimeMillis(); - result.addAll(parseValues(element.getAsJsonObject()).stream().map(kv -> new BaseAttributeKvEntry(kv, ts)).collect(Collectors.toList())); - return result; + return new HashSet<>(parseValues(element.getAsJsonObject()).stream().map(kv -> new BaseAttributeKvEntry(kv, ts)).toList()); + } + + public static Set convertToAttributes(JsonElement element, long ts) { + return new HashSet<>(parseValues(element.getAsJsonObject()).stream().map(kv -> new BaseAttributeKvEntry(kv, ts)).toList()); } private static List parseValues(JsonObject valuesObject) { @@ -702,4 +703,5 @@ public class JsonConverter { return ""; } } + } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java index ed0b425c76..a97dc411b4 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java @@ -49,6 +49,7 @@ public class JsonUtils { } return json; } + public static JsonElement parse(Object value) { if (value instanceof Integer) { return new JsonPrimitive((Integer) value); @@ -67,7 +68,7 @@ public class JsonUtils { } } - public static JsonObject convertToJsonObject(Map map) { + public static JsonObject convertToJsonObject(Map map) { JsonObject jsonObject = new JsonObject(); for (Map.Entry entry : map.entrySet()) { jsonObject.add(entry.getKey(), parse(entry.getValue())); @@ -75,4 +76,5 @@ public class JsonUtils { return jsonObject; } + } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java index 73368d40f5..f1fe1affc6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java @@ -89,6 +89,7 @@ public abstract class AbstractTbMsgPushNode { entityBody.put("kv", dataJson); + entityBody.put("ts", msg.getMetaDataTs()); entityBody.put(SCOPE, getScope(metadata)); if (EdgeEventActionType.POST_ATTRIBUTES.equals(actionType)) { entityBody.put("isPostAttributes", true); From d9dad36c54179b42d6cad4b66c82e6cab61436a8 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 15 Jan 2025 16:19:56 +0200 Subject: [PATCH 090/416] Send alarm_ack and alarm_clear only by 'Push to Edge' node --- .../edge/EdgeEventSourcingListener.java | 3 ++ .../server/edge/AlarmEdgeTest.java | 33 +++++-------------- .../engine/edge/AbstractTbMsgPushNode.java | 12 +++++-- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 9317c51a64..02b2346e64 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -135,6 +135,9 @@ public class EdgeEventSourcingListener { if (EntityType.DEVICE.equals(event.getEntityId().getEntityType()) && ActionType.ASSIGNED_TO_TENANT.equals(event.getActionType())) { return; } + if (EntityType.ALARM.equals(event.getEntityId().getEntityType())) { + return; + } try { if (event.getEntityId().getEntityType().equals(EntityType.RULE_CHAIN) && event.getEdgeId() != null && event.getActionType().equals(ActionType.ASSIGNED_TO_EDGE)) { try { diff --git a/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java index 239cb87ab4..0ceebf1cf4 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java @@ -94,44 +94,26 @@ public class AlarmEdgeTest extends AbstractEdgeTest { Alarm savedAlarm = doPost("/api/alarm", alarm, Alarm.class); edgeImitator.ignoreType(AlarmCommentUpdateMsg.class); - // ack alarm + // ack alarm - send only by using push to edge node edgeImitator.expectMessageAmount(1); doPost("/api/alarm/" + savedAlarm.getUuidId() + "/ack"); - Assert.assertTrue(edgeImitator.waitForMessages()); - AbstractMessage latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof AlarmUpdateMsg); - AlarmUpdateMsg alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; - Assert.assertEquals(UpdateMsgType.ALARM_ACK_RPC_MESSAGE, alarmUpdateMsg.getMsgType()); - Alarm alarmMsg = JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); - Assert.assertNotNull(alarmMsg); - Assert.assertEquals(savedAlarm.getType(), alarmMsg.getType()); - Assert.assertEquals(savedAlarm.getName(), alarmMsg.getName()); - Assert.assertEquals(AlarmStatus.ACTIVE_ACK, alarmMsg.getStatus()); + Assert.assertFalse(edgeImitator.waitForMessages(5)); - // clear alarm + // clear alarm - send only by using push to edge node edgeImitator.expectMessageAmount(1); doPost("/api/alarm/" + savedAlarm.getUuidId() + "/clear"); - Assert.assertTrue(edgeImitator.waitForMessages()); - latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof AlarmUpdateMsg); - alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; - Assert.assertEquals(UpdateMsgType.ALARM_CLEAR_RPC_MESSAGE, alarmUpdateMsg.getMsgType()); - alarmMsg = JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); - Assert.assertNotNull(alarmMsg); - Assert.assertEquals(savedAlarm.getType(), alarmMsg.getType()); - Assert.assertEquals(savedAlarm.getName(), alarmMsg.getName()); - Assert.assertEquals(AlarmStatus.CLEARED_ACK, alarmMsg.getStatus()); + Assert.assertFalse(edgeImitator.waitForMessages(5)); // delete alarm edgeImitator.expectMessageAmount(1); doDelete("/api/alarm/" + savedAlarm.getUuidId()) .andExpect(status().isOk()); Assert.assertTrue(edgeImitator.waitForMessages()); - latestMessage = edgeImitator.getLatestMessage(); + AbstractMessage latestMessage = edgeImitator.getLatestMessage(); Assert.assertTrue(latestMessage instanceof AlarmUpdateMsg); - alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; + AlarmUpdateMsg alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, alarmUpdateMsg.getMsgType()); - alarmMsg = JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); + Alarm alarmMsg = JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); Assert.assertNotNull(alarmMsg); Assert.assertEquals(savedAlarm.getType(), alarmMsg.getType()); Assert.assertEquals(savedAlarm.getName(), alarmMsg.getName()); @@ -252,4 +234,5 @@ public class AlarmEdgeTest extends AbstractEdgeTest { alarmComment.setCreatedTime(Uuids.unixTimestamp(uuid)); return alarmComment; } + } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java index 73368d40f5..8ed0313d32 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java @@ -39,6 +39,8 @@ import java.util.UUID; import static org.thingsboard.server.common.data.msg.TbMsgType.ACTIVITY_EVENT; import static org.thingsboard.server.common.data.msg.TbMsgType.ALARM; +import static org.thingsboard.server.common.data.msg.TbMsgType.ALARM_ACK; +import static org.thingsboard.server.common.data.msg.TbMsgType.ALARM_CLEAR; import static org.thingsboard.server.common.data.msg.TbMsgType.ATTRIBUTES_DELETED; import static org.thingsboard.server.common.data.msg.TbMsgType.ATTRIBUTES_UPDATED; import static org.thingsboard.server.common.data.msg.TbMsgType.CONNECT_EVENT; @@ -81,6 +83,9 @@ public abstract class AbstractTbMsgPushNode metadata = msg.getMetaData().getData(); EdgeEventActionType actionType = getEdgeEventActionTypeByMsgType(msg); @@ -164,8 +169,9 @@ public abstract class AbstractTbMsgPushNode Date: Wed, 15 Jan 2025 17:22:14 +0200 Subject: [PATCH 091/416] Change ts in test to be for sure condifent --- .../test/java/org/thingsboard/server/edge/DeviceEdgeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java index 11bc053bb5..e4f9080686 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java @@ -490,7 +490,7 @@ public class DeviceEdgeTest extends AbstractEdgeTest { @Test public void testSendOutdatedAttributeToCloud() throws Exception { - long ts = System.currentTimeMillis(); + long ts = System.currentTimeMillis() - 5; Device device = saveDeviceOnCloudAndVerifyDeliveryToEdge(); edgeImitator.expectResponsesAmount(1); From 6923639dc59c55c59556877380fc30d0acaf657a Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 15 Jan 2025 17:42:22 +0200 Subject: [PATCH 092/416] Fix AlarmControllerTest --- .../controller/AlarmControllerTest.java | 28 +++++++++---------- .../import-dialog.component.html | 3 +- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java index 2f264d49ff..6bd8396ceb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java @@ -169,7 +169,7 @@ public class AlarmControllerTest extends AbstractControllerTest { foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundAlarm, customerDevice, tenantId, - customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ACK, 1, 1, 1); + customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ACK, 1, 0, 1); Mockito.reset(tbClusterService, auditLogService); alarm = updatedAlarm; @@ -183,7 +183,7 @@ public class AlarmControllerTest extends AbstractControllerTest { foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundAlarm, customerDevice, tenantId, - customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_CLEAR, 1, 1, 1); + customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_CLEAR, 1, 0, 1); Mockito.reset(tbClusterService, auditLogService); alarm = updatedAlarm; @@ -197,7 +197,7 @@ public class AlarmControllerTest extends AbstractControllerTest { foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundAlarm, customerDevice, tenantId, - customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED, 1, 1, 1); + customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED, 1, 0, 1); Mockito.reset(tbClusterService, auditLogService); alarm = updatedAlarm; @@ -211,7 +211,7 @@ public class AlarmControllerTest extends AbstractControllerTest { foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundAlarm, customerDevice, tenantId, - customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_UNASSIGNED, 1, 1, 1); + customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_UNASSIGNED, 1, 0, 1); Mockito.reset(tbClusterService, auditLogService); } @@ -321,7 +321,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.CLEARED_UNACK, foundAlarm.getStatus()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_CLEAR); } @@ -337,7 +337,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.CLEARED_UNACK, foundAlarm.getStatus()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_CLEAR); } @@ -354,7 +354,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.ACTIVE_ACK, foundAlarm.getStatus()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_ACK); } @@ -434,7 +434,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(tenantAdminUserId, foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED); } @@ -465,7 +465,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(tenantAdminUserId, foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED); logout(); @@ -482,7 +482,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(customerUserId, foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_ASSIGNED); } @@ -500,7 +500,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(tenantAdminUserId, foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED); beforeAssignmentTs = System.currentTimeMillis(); @@ -511,7 +511,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertNull(foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_UNASSIGNED); } @@ -529,7 +529,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(tenantAdminUserId, foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED); logout(); @@ -545,7 +545,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertNull(foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_UNASSIGNED); } diff --git a/ui-ngx/src/app/shared/import-export/import-dialog.component.html b/ui-ngx/src/app/shared/import-export/import-dialog.component.html index 8bbe1599c4..76e6f8efac 100644 --- a/ui-ngx/src/app/shared/import-export/import-dialog.component.html +++ b/ui-ngx/src/app/shared/import-export/import-dialog.component.html @@ -63,8 +63,7 @@ {{ 'action.cancel' | translate }} From cad35a969e3db4894554f24cb72797b0356ba621 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Wed, 15 Jan 2025 17:48:53 +0200 Subject: [PATCH 093/416] Save time series strategies: add tests for persistence strategies --- .../DeduplicatePersistenceStrategyTest.java | 147 ++++++++++++++++++ ...OnEveryMessagePersistenceStrategyTest.java | 48 ++++++ .../strategy/PersistenceStrategyTest.java | 55 +++++++ .../strategy/SkipPersistenceStrategyTest.java | 48 ++++++ 4 files changed, 298 insertions(+) create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java new file mode 100644 index 0000000000..3f57a7f3df --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java @@ -0,0 +1,147 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry.strategy; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class DeduplicatePersistenceStrategyTest { + + final int deduplicationIntervalSecs = 10; + + DeduplicatePersistenceStrategy strategy; + + @BeforeEach + void setup() { + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + } + + @Test + void shouldThrowWhenDeduplicationIntervalIsLessThanOneSecond() { + assertThatThrownBy(() -> new DeduplicatePersistenceStrategy(0)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Deduplication interval must be at least 1 second(s), was 0 second(s)"); + } + + @Test + void shouldReturnTrueForFirstCallInInterval() { + long ts = 1_000_000L; + UUID originator = UUID.randomUUID(); + + assertThat(strategy.shouldPersist(ts, originator)).isTrue(); + } + + @Test + void shouldReturnFalseForSubsequentCallsInInterval() { + long baseTs = 1_000_000L; + UUID originator = UUID.randomUUID(); + + // Initial call should return true + assertThat(strategy.shouldPersist(baseTs, originator)).isTrue(); + + // Subsequent call within the same interval should return false for the same originator + long withinSameIntervalTs = baseTs + 1000L; + assertThat(strategy.shouldPersist(withinSameIntervalTs, originator)).isFalse(); + } + + @Test + void shouldHandleMultipleOriginatorsIndependently() { + long baseTs = 1_000_000L; + UUID originator1 = UUID.randomUUID(); + UUID originator2 = UUID.randomUUID(); + + // First call for different originators in the same interval should return true independently + assertThat(strategy.shouldPersist(baseTs, originator1)).isTrue(); + assertThat(strategy.shouldPersist(baseTs, originator2)).isTrue(); + + // Subsequent calls for the same originators within the same interval should return false + assertThat(strategy.shouldPersist(baseTs + 500L, originator1)).isFalse(); + assertThat(strategy.shouldPersist(baseTs + 500L, originator2)).isFalse(); + } + + @Test + void shouldHandleEdgeCaseTimestamps() { + long minTs = Long.MIN_VALUE; + long maxTs = Long.MAX_VALUE; + UUID originator = UUID.randomUUID(); + + assertThat(strategy.shouldPersist(minTs, originator)).isTrue(); + assertThat(strategy.shouldPersist(minTs + 1L, originator)).isFalse(); + + assertThat(strategy.shouldPersist(maxTs, originator)).isTrue(); + assertThat(strategy.shouldPersist(maxTs - 1L, originator)).isFalse(); + } + + @Test + void shouldResetDeduplicationAtIntervalBoundaries() { + UUID originator = UUID.randomUUID(); + + // check 1st interval + long firstIntervalStart = 0L; + long firstIntervalEnd = firstIntervalStart + Duration.ofSeconds(deduplicationIntervalSecs).toMillis() - 1L; + long firstIntervalMiddle = calculateMiddle(firstIntervalStart, firstIntervalEnd); + + assertThat(strategy.shouldPersist(firstIntervalStart, originator)).isTrue(); + assertThat(strategy.shouldPersist(firstIntervalStart + 1, originator)).isFalse(); + assertThat(strategy.shouldPersist(firstIntervalMiddle, originator)).isFalse(); + assertThat(strategy.shouldPersist(firstIntervalEnd - 1, originator)).isFalse(); + assertThat(strategy.shouldPersist(firstIntervalEnd, originator)).isFalse(); + + // check 2nd interval + long secondIntervalStart = firstIntervalEnd + 1L; + long secondIntervalEnd = secondIntervalStart + Duration.ofSeconds(deduplicationIntervalSecs).toMillis() - 1L; + long secondIntervalMiddle = calculateMiddle(secondIntervalStart, secondIntervalEnd); + + assertThat(strategy.shouldPersist(secondIntervalStart, originator)).isTrue(); + assertThat(strategy.shouldPersist(secondIntervalStart + 1, originator)).isFalse(); + assertThat(strategy.shouldPersist(secondIntervalMiddle, originator)).isFalse(); + assertThat(strategy.shouldPersist(secondIntervalEnd - 1, originator)).isFalse(); + assertThat(strategy.shouldPersist(secondIntervalEnd, originator)).isFalse(); + } + + @Test + void shouldHandleMultipleOriginatorsOverMultipleIntervals() { + UUID originator1 = UUID.randomUUID(); + UUID originator2 = UUID.randomUUID(); + long baseTs = 0L; + + // First interval for both originators + assertThat(strategy.shouldPersist(baseTs, originator1)).isTrue(); + assertThat(strategy.shouldPersist(baseTs, originator2)).isTrue(); + + // Move to the next interval + long nextIntervalTs = baseTs + Duration.ofSeconds(10).toMillis(); + + // Each originator should be allowed again in the new interval + assertThat(strategy.shouldPersist(nextIntervalTs, originator1)).isTrue(); + assertThat(strategy.shouldPersist(nextIntervalTs, originator2)).isTrue(); + + // Subsequent calls in the same new interval should return false + assertThat(strategy.shouldPersist(nextIntervalTs + 500L, originator1)).isFalse(); + assertThat(strategy.shouldPersist(nextIntervalTs + 500L, originator2)).isFalse(); + } + + private static long calculateMiddle(long start, long end) { + return start + (end - start) / 2; + } + +} diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java new file mode 100644 index 0000000000..125da3a495 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry.strategy; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.UUID; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class OnEveryMessagePersistenceStrategyTest { + + @ParameterizedTest + @MethodSource("edgeCaseProvider") + void shouldAlwaysReturnTrueForAnyInput(long timestamp, UUID originator) { + var onEveryMessage = OnEveryMessagePersistenceStrategy.getInstance(); + assertThat(onEveryMessage.shouldPersist(timestamp, originator)).isTrue(); + } + + private static Stream edgeCaseProvider() { + return Stream.of( + Arguments.of(Long.MIN_VALUE, new UUID(0L, 0L)), + Arguments.of(Long.MAX_VALUE, new UUID(Long.MAX_VALUE, Long.MAX_VALUE)), + Arguments.of(0L, new UUID(0L, 0L)), + Arguments.of(-1L, new UUID(-1L, -1L)), + Arguments.of(1L, new UUID(1L, 1L)), + Arguments.of(42L, UUID.randomUUID()), + Arguments.of(1000L, null) + ); + } + +} diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java new file mode 100644 index 0000000000..71a8eabed5 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry.strategy; + +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class PersistenceStrategyTest { + + @Test + void testOnEveryMessageReturnsCorrectInstance() { + PersistenceStrategy strategy = PersistenceStrategy.onEveryMessage(); + assertThat(strategy) + .isNotNull() + .isInstanceOf(OnEveryMessagePersistenceStrategy.class); + } + + @Test + void testDeduplicateReturnsCorrectInstance() { + int validDeduplicationIntervalSecs = 5; + PersistenceStrategy strategy = PersistenceStrategy.deduplicate(validDeduplicationIntervalSecs); + assertThat(strategy) + .isNotNull() + .isInstanceOf(DeduplicatePersistenceStrategy.class); + + long actualDeduplicationIntervalMillis = (long) ReflectionTestUtils.getField(strategy, "deduplicationIntervalMillis"); + assertThat(actualDeduplicationIntervalMillis).isEqualTo(Duration.ofSeconds(validDeduplicationIntervalSecs).toMillis()); + } + + @Test + void testSkipReturnsCorrectInstance() { + PersistenceStrategy strategy = PersistenceStrategy.skip(); + assertThat(strategy) + .isNotNull() + .isInstanceOf(SkipPersistenceStrategy.class); + } + +} diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java new file mode 100644 index 0000000000..1a63ce7460 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry.strategy; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.UUID; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class SkipPersistenceStrategyTest { + + @ParameterizedTest + @MethodSource("edgeCaseProvider") + void shouldAlwaysReturnFalseForAnyInput(long timestamp, UUID originator) { + var skipStrategy = SkipPersistenceStrategy.getInstance(); + assertThat(skipStrategy.shouldPersist(timestamp, originator)).isFalse(); + } + + private static Stream edgeCaseProvider() { + return Stream.of( + Arguments.of(Long.MIN_VALUE, new UUID(0L, 0L)), + Arguments.of(Long.MAX_VALUE, new UUID(Long.MAX_VALUE, Long.MAX_VALUE)), + Arguments.of(0L, new UUID(0L, 0L)), + Arguments.of(-1L, new UUID(-1L, -1L)), + Arguments.of(1L, new UUID(1L, 1L)), + Arguments.of(42L, UUID.randomUUID()), + Arguments.of(1000L, null) + ); + } + +} From 3aac81745f5d52f1d26ac2e679ed8d040e14f9cb Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Thu, 16 Jan 2025 12:58:22 +0200 Subject: [PATCH 094/416] Save time series strategies: add test for changes in entity view service --- .../DefaultTbEntityViewServiceTest.java | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java new file mode 100644 index 0000000000..b36890f775 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java @@ -0,0 +1,109 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.entityview; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.objects.AttributesEntityView; +import org.thingsboard.server.common.data.objects.TelemetryEntityView; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; + +import java.util.List; +import java.util.UUID; + +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; + +@ExtendWith(MockitoExtension.class) +class DefaultTbEntityViewServiceTest { + + final TenantId tenantId = TenantId.fromUUID(UUID.fromString("f09c8180-686c-11ef-9471-a71d33080e9c")); + final EntityId entityId = DeviceId.fromString("782aaab0-c7a8-11ef-a668-79582e785d5f"); + + @Mock + EntityViewService entityViewService; + @Mock + AttributesService attributesService; + @Mock + TelemetrySubscriptionService tsSubService; + @Mock + TimeseriesService tsService; + + DefaultTbEntityViewService defaultTbEntityViewService; + + @BeforeEach + void setup() { + defaultTbEntityViewService = new DefaultTbEntityViewService(entityViewService, attributesService, tsSubService, tsService); + } + + @Test + void shouldNotSaveTimeseriesWhenCopyingLatestToEntityView() throws Exception { + // GIVEN + var entityView = new EntityView(new EntityViewId(UUID.randomUUID())); + entityView.setTenantId(tenantId); + entityView.setEntityId(entityId); + entityView.setKeys(new TelemetryEntityView(List.of("temperature"), new AttributesEntityView())); + + List latest = List.of(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))); + + given(tsService.findAll(eq(tenantId), eq(entityId), anyList())).willReturn(immediateFuture(latest)); + + // WHEN + defaultTbEntityViewService.updateEntityViewAttributes(tenantId, entityView, null, null); + + // THEN + var captor = ArgumentCaptor.forClass(TimeseriesSaveRequest.class); + then(tsSubService).should().saveTimeseries(captor.capture()); + + var expectedCopyLatestRequest = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .entityId(entityView.getId()) + .entries(latest) + .ttl(0L) + .saveTimeseries(false) + .saveLatest(true) + .sendWsUpdate(true) + .build(); + + var actualCopyLatestRequest = captor.getValue(); + + assertThat(actualCopyLatestRequest) + .usingRecursiveComparison() + .ignoringFields("callback") + .isEqualTo(expectedCopyLatestRequest); + } + +} From 0b277661bdba594b2d22b04914a30cde039b1c23 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 17 Jan 2025 12:36:52 +0200 Subject: [PATCH 095/416] Save time series strategies: add tests for changes in save time series node --- .../engine/telemetry/TbMsgTimeseriesNode.java | 6 +- .../TbMsgTimeseriesNodeConfiguration.java | 11 +- .../DeduplicatePersistenceStrategy.java | 5 + .../telemetry/TbMsgTimeseriesNodeTest.java | 365 ++++++++++++++++-- 4 files changed, 342 insertions(+), 45 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index db2421e307..422065f81f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -90,11 +90,11 @@ public class TbMsgTimeseriesNode implements TbNode { onTenantProfileUpdate(ctx.getTenantProfile()); persistenceSettings = config.getPersistenceSettings(); if (persistenceSettings == null) { - throw new TbNodeException("Persistence settings cannot be null!", true); + throw new TbNodeException("Persistence settings cannot be null", true); } } - void onTenantProfileUpdate(TenantProfile tenantProfile) { + private void onTenantProfileUpdate(TenantProfile tenantProfile) { DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); } @@ -193,7 +193,7 @@ public class TbMsgTimeseriesNode implements TbNode { boolean hasChanges = false; switch (fromVersion) { case 0: - if (oldConfiguration.has("persistenceSettings")) { + if (oldConfiguration.has("persistenceSettings") && !oldConfiguration.has("skipLatestPersistence")) { break; } hasChanges = true; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java index 8be8cac24e..f702ecc6da 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java @@ -16,6 +16,7 @@ package org.thingsboard.rule.engine.telemetry; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -67,11 +68,15 @@ public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration Sets.newConcurrentHashSet()); } + @JsonProperty("deduplicationIntervalSecs") + public long getDeduplicationIntervalSecs() { + return Duration.ofMillis(deduplicationIntervalMillis).toSeconds(); + } + @Override public boolean shouldPersist(long ts, UUID originatorUuid) { long intervalNumber = ts / deduplicationIntervalMillis; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index 71d060aab2..d0727b1a3c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; @@ -57,25 +58,30 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.assertArg; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("c8f34868-603a-4433-876a-7d356e5cf377")); private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("e5095e9a-04f4-44c9-b443-1cf1b97d3384")); - private final TenantProfileId TENANT_PROFILE_ID = new TenantProfileId(UUID.fromString("ab78dd78-83d0-43fa-869f-d42ec9ed1744")); + + private TenantProfile tenantProfile; private TbMsgTimeseriesNode node; private TbMsgTimeseriesNodeConfiguration config; - private long tenantProfileDefaultStorageTtl; @Mock private TbContext ctxMock; @@ -84,6 +90,17 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @BeforeEach public void setUp() throws TbNodeException { + tenantProfile = new TenantProfile(new TenantProfileId(UUID.fromString("ab78dd78-83d0-43fa-869f-d42ec9ed1744"))); + var tenantProfileConfiguration = new DefaultTenantProfileConfiguration(); + tenantProfileConfiguration.setDefaultStorageTtlDays(5); + var tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(tenantProfileConfiguration); + tenantProfile.setProfileData(tenantProfileData); + lenient().when(ctxMock.getTenantProfile()).thenReturn(tenantProfile); + + lenient().when(ctxMock.getTenantId()).thenReturn(TENANT_ID); + lenient().when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); + node = spy(new TbMsgTimeseriesNode()); config = new TbMsgTimeseriesNodeConfiguration().defaultConfiguration(); } @@ -95,18 +112,47 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(config.isUseServerTs()).isFalse(); } + @Test + public void whenInit_thenShouldAddTenantProfileListener() throws Exception { + // GIVEN-WHEN + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + // THEN + then(ctxMock).should().addTenantProfileListener(any()); + } + + @Test + public void givenPersistenceSettingsAreNull_whenInit_thenThrowsException() { + // GIVEN + config.setPersistenceSettings(null); + + // WHEN-THEN + assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) + .isInstanceOf(TbNodeException.class) + .matches(e -> ((TbNodeException) e).isUnrecoverable()) + .hasMessage("Persistence settings cannot be null"); + } + @ParameterizedTest @EnumSource(TbMsgType.class) public void givenMsgTypeAndEmptyMsgData_whenOnMsg_thenVerifyFailureMsg(TbMsgType msgType) throws TbNodeException { - init(); + // GIVEN + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + TbMsg msg = TbMsg.newMsg() .type(msgType) .originator(DEVICE_ID) .copyMetaData(TbMsgMetaData.EMPTY) .data(TbMsg.EMPTY_JSON_ARRAY) .build(); + + // WHEN node.onMsg(ctxMock, msg); + // THEN + then(ctxMock).should().addTenantProfileListener(any()); + then(ctxMock).should().getTenantProfile(); + ArgumentCaptor throwableCaptor = ArgumentCaptor.forClass(Throwable.class); verify(ctxMock).tellFailure(eq(msg), throwableCaptor.capture()); @@ -120,9 +166,11 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenTtlFromConfigIsZeroAndUseServiceTsIsTrue_whenOnMsg_thenSaveTimeseriesUsingTenantProfileDefaultTtl() throws TbNodeException { + public void givenTtlFromConfigIsZeroAndUseServerTsIsTrue_whenOnMsg_thenSaveTimeseriesUsingTenantProfileDefaultTtl() throws TbNodeException { + // GIVEN config.setUseServerTs(true); - init(); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); String data = """ { @@ -137,23 +185,28 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .data(data) .build(); - when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); - when(ctxMock.getTenantId()).thenReturn(TENANT_ID); doAnswer(invocation -> { TimeseriesSaveRequest request = invocation.getArgument(0); request.getCallback().onSuccess(null); return null; }).when(telemetryServiceMock).saveTimeseries(any(TimeseriesSaveRequest.class)); + // WHEN node.onMsg(ctxMock, msg); + // THEN + then(ctxMock).should().getTenantId(); + then(ctxMock).should().getTelemetryService(); + then(ctxMock).should().addTenantProfileListener(any()); + then(ctxMock).should().getTenantProfile(); + List expectedList = getTsKvEntriesListWithTs(data, System.currentTimeMillis()); verify(telemetryServiceMock).saveTimeseries(assertArg(request -> { assertThat(request.getTenantId()).isEqualTo(TENANT_ID); assertThat(request.getCustomerId()).isNull(); assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).usingRecursiveFieldByFieldElementComparatorIgnoringFields("ts").containsExactlyElementsOf(expectedList); - assertThat(request.getTtl()).isEqualTo(tenantProfileDefaultStorageTtl); + assertThat(request.getTtl()).isEqualTo(extractTtlAsSeconds(tenantProfile)); assertThat(request.isSaveLatest()).isTrue(); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); @@ -162,9 +215,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenSkipLatestPersistenceIsTrueAndTtlFromConfig_whenOnMsg_thenSaveTimeseriesUsingTtlFromConfig() throws TbNodeException { - long ttlFromConfig = 5L; - config.setDefaultTTL(ttlFromConfig); + public void givenSkipLatestPersistenceSettingsAndTtlFromConfig_whenOnMsg_thenSaveTimeseriesUsingTtlFromConfig() throws TbNodeException { + // GIVEN + config.setDefaultTTL(10L); var timeseriesStrategy = PersistenceStrategy.onEveryMessage(); var latestStrategy = PersistenceStrategy.skip(); @@ -172,7 +225,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { var persistenceSettings = new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced(timeseriesStrategy, latestStrategy, webSockets); config.setPersistenceSettings(persistenceSettings); - init(); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); String data = """ { @@ -189,23 +242,28 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .data(data) .build(); - when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); - when(ctxMock.getTenantId()).thenReturn(TENANT_ID); doAnswer(invocation -> { TimeseriesSaveRequest request = invocation.getArgument(0); request.getCallback().onSuccess(null); return null; }).when(telemetryServiceMock).saveTimeseries(any(TimeseriesSaveRequest.class)); + // WHEN node.onMsg(ctxMock, msg); + // THEN + then(ctxMock).should().getTenantId(); + then(ctxMock).should().getTelemetryService(); + then(ctxMock).should().addTenantProfileListener(any()); + then(ctxMock).should().getTenantProfile(); + List expectedList = getTsKvEntriesListWithTs(data, ts); verify(telemetryServiceMock).saveTimeseries(assertArg(request -> { assertThat(request.getTenantId()).isEqualTo(TENANT_ID); assertThat(request.getCustomerId()).isNull(); assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).containsExactlyElementsOf(expectedList); - assertThat(request.getTtl()).isEqualTo(ttlFromConfig); + assertThat(request.getTtl()).isEqualTo(config.getDefaultTTL()); assertThat(request.isSaveTimeseries()).isTrue(); assertThat(request.isSaveLatest()).isFalse(); assertThat(request.isSendWsUpdate()).isTrue(); @@ -218,11 +276,10 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @ParameterizedTest @MethodSource public void givenTtlFromConfigAndTtlFromMd_whenOnMsg_thenVerifyTtl(String ttlFromMd, long ttlFromConfig, long expectedTtl) throws TbNodeException { + // GIVEN config.setDefaultTTL(ttlFromConfig); - init(); - when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); - when(ctxMock.getTenantId()).thenReturn(TENANT_ID); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); String data = """ { @@ -238,8 +295,11 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .copyMetaData(metadata) .data(data) .build(); + + // WHEN node.onMsg(ctxMock, msg); + // THEN verify(telemetryServiceMock).saveTimeseries(assertArg(request -> { assertThat(request.getTenantId()).isEqualTo(TENANT_ID); assertThat(request.getCustomerId()).isNull(); @@ -262,26 +322,6 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { ); } - private void init() throws TbNodeException { - var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - var tenantProfile = getTenantProfile(); - when(ctxMock.getTenantProfile()).thenReturn(tenantProfile); - tenantProfile.getProfileConfiguration().ifPresent(profileConfiguration -> - tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(profileConfiguration.getDefaultStorageTtlDays())); - node.init(ctxMock, configuration); - verify(ctxMock).addTenantProfileListener(any()); - } - - private TenantProfile getTenantProfile() { - var tenantProfile = new TenantProfile(TENANT_PROFILE_ID); - var tenantProfileData = new TenantProfileData(); - var tenantProfileConfiguration = new DefaultTenantProfileConfiguration(); - tenantProfileConfiguration.setDefaultStorageTtlDays(5); - tenantProfileData.setConfiguration(tenantProfileConfiguration); - tenantProfile.setProfileData(tenantProfileData); - return tenantProfile; - } - private static List getTsKvEntriesListWithTs(String data, long ts) { Map> tsKvMap = JsonConverter.convertToTelemetry(JsonParser.parseString(data), ts); List expectedList = new ArrayList<>(); @@ -293,6 +333,253 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { return expectedList; } + @Test + public void givenOnEveryMessagePersistenceSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.OnEveryMessage()); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", "123"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = TimeseriesSaveRequest.builder() + .tenantId(TENANT_ID) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) + .ttl(extractTtlAsSeconds(tenantProfile)) + .saveTimeseries(true) + .saveLatest(true) + .sendWsUpdate(true) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(1)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(2)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + } + + @Test + public void givenDeduplicatePersistenceSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistThisMessageOnlyFirstTime() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Deduplicate(10)); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", "123"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = TimeseriesSaveRequest.builder() + .tenantId(TENANT_ID) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) + .ttl(extractTtlAsSeconds(tenantProfile)) + .saveTimeseries(true) + .saveLatest(true) + .sendWsUpdate(true) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + + clearInvocations(telemetryServiceMock, ctxMock); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(never()).saveTimeseries(any()); + } + + @Test + public void givenWebsocketsOnlyPersistenceSettingsAndSameMessageTwoTimes_whenOnMsg_thenSendsOnlyWsUpdateTwoTimes() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.WebSocketsOnly()); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", "123"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = TimeseriesSaveRequest.builder() + .tenantId(TENANT_ID) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) + .ttl(extractTtlAsSeconds(tenantProfile)) + .saveTimeseries(false) + .saveLatest(false) + .sendWsUpdate(true) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(1)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(2)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + } + + @Test + public void givenAdvancedPersistenceSettingsWithOnEveryMessageStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced( + PersistenceStrategy.onEveryMessage(), + PersistenceStrategy.onEveryMessage(), + PersistenceStrategy.onEveryMessage() + )); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", "123"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = TimeseriesSaveRequest.builder() + .tenantId(TENANT_ID) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) + .ttl(extractTtlAsSeconds(tenantProfile)) + .saveTimeseries(true) + .saveLatest(true) + .sendWsUpdate(true) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(1)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(2)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + } + + @Test + public void givenAdvancedPersistenceSettingsWithDifferentDeduplicateStrategyForEachAction_whenOnMsg_thenEvaluatesStrategiesForEachActionsIndependently() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced( + PersistenceStrategy.deduplicate(1), + PersistenceStrategy.deduplicate(2), + PersistenceStrategy.deduplicate(3) + )); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + long ts1 = 500L; + long ts2 = 1500L; + long ts3 = 2500L; + + // WHEN-THEN + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts1)))) + .build()); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> { + assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); + assertThat(actualSaveRequest.isSaveLatest()).isTrue(); + assertThat(actualSaveRequest.isSendWsUpdate()).isTrue(); + } + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts2)))) + .build()); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> { + assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); + assertThat(actualSaveRequest.isSaveLatest()).isFalse(); + assertThat(actualSaveRequest.isSendWsUpdate()).isFalse(); + } + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts3)))) + .build()); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> { + assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); + assertThat(actualSaveRequest.isSaveLatest()).isTrue(); + assertThat(actualSaveRequest.isSendWsUpdate()).isFalse(); + } + )); + } + + @Test + public void givenAdvancedPersistenceSettingsWithSkipStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenSkipsSameMessageTwoTimes() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced( + PersistenceStrategy.skip(), + PersistenceStrategy.skip(), + PersistenceStrategy.skip() + )); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", "123"))) + .build(); + + // WHEN-THEN + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(never()).saveTimeseries(any()); + then(ctxMock).should(times(1)).tellSuccess(msg); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(never()).saveTimeseries(any()); + then(ctxMock).should(times(2)).tellSuccess(msg); + } + + private static long extractTtlAsSeconds(TenantProfile tenantProfile) { + return TimeUnit.DAYS.toSeconds(tenantProfile.getDefaultProfileConfiguration().getDefaultStorageTtlDays()); + } + @Override protected TbNode getTestNode() { return node; From 71d43f3af2f3fbb1ce1fccd3a2424e3d3f86bd0e Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 17 Jan 2025 15:24:06 +0200 Subject: [PATCH 096/416] Save time series strategies: update configurationVersion for save time series node in rule chain JSONs --- .../data/json/edge/rule_chains/edge_root_rule_chain.json | 1 + .../json/tenant/device_profile/rule_chain_template.json | 1 + .../main/data/json/tenant/rule_chains/root_rule_chain.json | 1 + monitoring/src/main/resources/root_rule_chain.json | 6 +++--- .../src/test/resources/MqttRuleNodeTestMetadata.json | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json index e7b4b93e98..e663ff779d 100644 --- a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json +++ b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json @@ -33,6 +33,7 @@ }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, diff --git a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json index 37cc226a6e..325e01003f 100644 --- a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json +++ b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json @@ -19,6 +19,7 @@ }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, diff --git a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json index 7893a64fd2..4dc202d740 100644 --- a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json @@ -18,6 +18,7 @@ }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, diff --git a/monitoring/src/main/resources/root_rule_chain.json b/monitoring/src/main/resources/root_rule_chain.json index a48fd6bf35..eda3e44e1b 100644 --- a/monitoring/src/main/resources/root_rule_chain.json +++ b/monitoring/src/main/resources/root_rule_chain.json @@ -21,7 +21,7 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "singletonMode": false, - "configurationVersion": 0, + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, @@ -277,7 +277,7 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "singletonMode": false, - "configurationVersion": 0, + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, @@ -315,7 +315,7 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries with TTL", "singletonMode": false, - "configurationVersion": 0, + "configurationVersion": 1, "configuration": { "defaultTTL": 180, "useServerTs": false, diff --git a/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json b/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json index adcb618cbe..c495e590cd 100644 --- a/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json +++ b/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json @@ -36,7 +36,7 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "save timeseries", "singletonMode": false, - "configurationVersion": 0, + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, From 1bbb88b826e439f68970d9d5ff1340af3d8158a7 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 20 Jan 2025 16:21:24 +0200 Subject: [PATCH 097/416] Revert import-dialog back to default --- .../src/app/shared/import-export/import-dialog.component.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/shared/import-export/import-dialog.component.html b/ui-ngx/src/app/shared/import-export/import-dialog.component.html index 76e6f8efac..8bbe1599c4 100644 --- a/ui-ngx/src/app/shared/import-export/import-dialog.component.html +++ b/ui-ngx/src/app/shared/import-export/import-dialog.component.html @@ -63,7 +63,8 @@ {{ 'action.cancel' | translate }} From 12233475875c6e83f1bae1c6ab7c186a1115a27e Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 20 Jan 2025 16:38:03 +0200 Subject: [PATCH 098/416] Fix testSendOutdatedAttributeToCloud --- .../test/java/org/thingsboard/server/edge/DeviceEdgeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java index e4f9080686..060e641b93 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java @@ -490,7 +490,7 @@ public class DeviceEdgeTest extends AbstractEdgeTest { @Test public void testSendOutdatedAttributeToCloud() throws Exception { - long ts = System.currentTimeMillis() - 5; + long ts = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1); Device device = saveDeviceOnCloudAndVerifyDeliveryToEdge(); edgeImitator.expectResponsesAmount(1); From b898ca4d15bbb057fd61406afc4b6b2843e73bcf Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 20 Jan 2025 17:36:07 +0200 Subject: [PATCH 099/416] UI: Add time unit selector and improved save ts rule node --- .../action/timeseries-config.component.html | 60 +++--- .../action/timeseries-config.component.ts | 1 - .../common/common-rule-node-config.module.ts | 7 +- .../common/time-unit-input.component.html | 43 ++++ .../common/time-unit-input.component.ts | 184 ++++++++++++++++++ .../assets/locale/locale.constant-en_US.json | 7 +- 6 files changed, 264 insertions(+), 38 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html index 31e827d331..aa70914fa9 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html @@ -16,35 +16,33 @@ -->

- - rule-node-config.default-ttl - - - help - - - {{ 'rule-node-config.default-ttl-required' | translate }} - - - {{ 'rule-node-config.min-default-ttl-message' | translate }} - - -
-
- - {{ 'rule-node-config.use-server-ts' | translate }} - -
-
- - {{ 'rule-node-config.skip-latest-persistence' | translate }} - -
-
+
+ + + rule-node-config.advanced-settings + + + + + help + + +
+ + {{ 'rule-node-config.use-server-ts' | translate }} + +
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts index d480ec4d39..0c5283f54a 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts @@ -38,7 +38,6 @@ export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { protected onConfigurationSet(configuration: RuleNodeConfiguration) { this.timeseriesConfigForm = this.fb.group({ defaultTTL: [configuration ? configuration.defaultTTL : null, [Validators.required, Validators.min(0)]], - skipLatestPersistence: [configuration ? configuration.skipLatestPersistence : false, []], useServerTs: [configuration ? configuration.useServerTs : false, []] }); } diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/common-rule-node-config.module.ts b/ui-ngx/src/app/modules/home/components/rule-node/common/common-rule-node-config.module.ts index c72b9901c9..b6370be318 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/common-rule-node-config.module.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/common-rule-node-config.module.ts @@ -33,6 +33,7 @@ import { RelationsQueryConfigOldComponent } from './relations-query-config-old.c import { SelectAttributesComponent } from './select-attributes.component'; import { AlarmStatusSelectComponent } from './alarm-status-select.component'; import { ExampleHintComponent } from './example-hint.component'; +import { TimeUnitInputComponent } from './time-unit-input.component'; @NgModule({ declarations: [ @@ -50,7 +51,8 @@ import { ExampleHintComponent } from './example-hint.component'; RelationsQueryConfigOldComponent, SelectAttributesComponent, AlarmStatusSelectComponent, - ExampleHintComponent + ExampleHintComponent, + TimeUnitInputComponent ], imports: [ CommonModule, @@ -72,7 +74,8 @@ import { ExampleHintComponent } from './example-hint.component'; RelationsQueryConfigOldComponent, SelectAttributesComponent, AlarmStatusSelectComponent, - ExampleHintComponent + ExampleHintComponent, + TimeUnitInputComponent ] }) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html new file mode 100644 index 0000000000..79caab4cb7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html @@ -0,0 +1,43 @@ + +
+ + {{ labelText }} + +
+ +
+ + {{ requiredText }} + + + {{ minErrorText }} + + + {{ maxErrorText }} + + + + rule-node-config.units + + @for (timeUnit of timeUnits; track timeUnit) { + {{ timeUnitTranslations.get(timeUnit) | translate }} + } + + +
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts new file mode 100644 index 0000000000..3dc0c00124 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts @@ -0,0 +1,184 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, DestroyRef, forwardRef, Input, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, Validators +} from '@angular/forms'; +import { TimeUnit, timeUnitTranslations } from '../rule-node-config.models'; +import { isDefinedAndNotNull, isNumeric } from '@core/utils'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { coerceBoolean, coerceNumber } from '@shared/decorators/coercion'; +import { DAY, HOUR, MINUTE, SECOND } from '@shared/models/time/time.models'; + +interface TimeUnitInputModel { + time: number; + timeUnit: TimeUnit +} + +@Component({ + selector: 'tb-time-unit-input', + templateUrl: './time-unit-input.component.html', + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => TimeUnitInputComponent), + multi: true + },{ + provide: NG_VALIDATORS, + useExisting: forwardRef(() => TimeUnitInputComponent), + multi: true + }] +}) +export class TimeUnitInputComponent implements ControlValueAccessor, Validator, OnInit { + + @Input() + labelText: string; + + @Input() + @coerceBoolean() + required: boolean; + + @Input() + requiredText: string; + + @Input() + minErrorText: string; + + @Input() + @coerceNumber() + maxTime: number; + + @Input() + maxErrorText: string; + + timeUnits = Object.values(TimeUnit).filter(item => item !== TimeUnit.MILLISECONDS) as TimeUnit[]; + + timeUnitTranslations = timeUnitTranslations; + + timeInputForm = this.fb.group({ + time: [0, Validators.min(0)], + timeUnit: [TimeUnit.SECONDS] + }); + + private timeIntervalsInSec = new Map([ + [TimeUnit.DAYS, DAY/SECOND], + [TimeUnit.HOURS, HOUR/SECOND], + [TimeUnit.MINUTES, MINUTE/SECOND], + [TimeUnit.SECONDS, SECOND/SECOND], + ]); + + private modelValue: number; + + private propagateChange: (value: any) => void = () => {}; + + constructor(private fb: FormBuilder, + private destroyRef: DestroyRef) { + } + + ngOnInit() { + if(this.required || this.maxTime) { + const timeControl = this.timeInputForm.get('time'); + const validators = []; + if (this.required) { + validators.push(Validators.required); + } + if (this.maxTime) { + validators.push((control: AbstractControl) => + Validators.max(Math.floor(this.maxTime / this.timeIntervalsInSec.get(this.timeInputForm.get('timeUnit').value)))(control) + ); + } + + timeControl.setValidators(validators); + timeControl.updateValueAndValidity({ emitEvent: false }); + } + + this.timeInputForm.get('timeUnit').valueChanges.pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe(() => { + this.timeInputForm.get('time').updateValueAndValidity({onlySelf: true}); + this.timeInputForm.get('time').markAsTouched({onlySelf: true}); + }); + + this.timeInputForm.valueChanges.pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe(value => { + this.updatedModel(value); + }); + } + + registerOnChange(fn: any) { + this.propagateChange = fn; + } + + registerOnTouched(_fn: any) { + } + + setDisabledState(isDisabled: boolean) { + if (isDisabled) { + this.timeInputForm.disable({emitEvent: false}); + } else { + this.timeInputForm.enable({emitEvent: false}); + } + } + + writeValue(sec: number) { + if (sec !== this.modelValue) { + if (isDefinedAndNotNull(sec) && isNumeric(sec) && Number(sec) !== 0) { + this.timeInputForm.patchValue(this.parseTime(sec), {emitEvent: false}); + this.modelValue = sec; + } else { + this.timeInputForm.patchValue({ + time: 0, + timeUnit: TimeUnit.SECONDS + }, {emitEvent: false}); + this.modelValue = 0; + } + } + } + + validate(): ValidationErrors | null { + return this.timeInputForm.valid ? null : { + timeInput: false + }; + } + + private updatedModel(value: Partial) { + const time = value.time * this.timeIntervalsInSec.get(value.timeUnit); + if (this.modelValue !== time) { + this.modelValue = time; + this.propagateChange(time); + } + } + + private parseTime(value: number): TimeUnitInputModel { + for (const [timeUnit, timeValue] of this.timeIntervalsInSec) { + const calc = value / timeValue; + if (Number.isInteger(calc)) { + return { + time: calc, + timeUnit: timeUnit + } + } + } + } + +} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 2fb67adbba..b6aa720a53 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -4541,7 +4541,7 @@ "originator-entity": "Entity by name pattern", "clone-message": "Clone message", "transform": "Transform", - "default-ttl": "Default TTL in seconds", + "default-ttl": "Default TTL", "default-ttl-required": "Default TTL is required.", "default-ttl-hint": "Rule node will fetch Time-to-Live (TTL) value from the message metadata. If no value is present, it defaults to the TTL specified in the configuration. If the value is set to 0, the TTL from the tenant profile configuration will be applied.", "default-ttl-zero-hint": "TTL will not be applied if its value is set to 0.", @@ -4906,9 +4906,7 @@ "general-pattern-hint": "Use ${metadataKey} for value from metadata, $[messageKey] for value from message body.", "alarm-severity-pattern-hint": "Use ${metadataKey} for value from metadata, $[messageKey] for value from message body. Alarm severity should be system (CRITICAL, MAJOR etc.)", "output-node-name-hint": "The rule node name corresponds to the relation type of the output message, and it is used to forward messages to other rule nodes in the caller rule chain.", - "skip-latest-persistence": "Skip latest persistence", - "skip-latest-persistence-hint": "Rule node will not update values for incoming keys for the latest time series data. Useful for highly loaded use-cases to decrease the pressure on the DB.", - "use-server-ts": "Use server ts", + "use-server-ts": "Use server timestamp", "use-server-ts-hint": "Rule node will use the timestamp of message processing instead of the timestamp from the message. Useful for all sorts of sequential processing if you merge messages from multiple sources (devices, assets, etc).", "kv-map-pattern-hint": "All input fields support templatization. Use $[messageKey] to extract value from the message and ${metadataKey} to extract value from the metadata.", "kv-map-single-pattern-hint": "Input field support templatization. Use $[messageKey] to extract value from the message and ${metadataKey} to extract value from the metadata.", @@ -5067,6 +5065,7 @@ "request-timeout-required": "Request timeout is required", "request-timeout-min": "Min request timeout is 0", "request-timeout-hint": "The amount of time to wait in seconds for the request to complete before giving up and timing out. A value of 0 means infinity, and is not recommended.", + "units": "Units", "tell-failure-aws-lambda": "Tell Failure if AWS Lambda function execution raises exception", "tell-failure-aws-lambda-hint": "Rule node forces failure of message processing if AWS Lambda function execution raises exception.", "key-val": { From 5d07beb0dedcee4182f817130e21b5a20e9864be Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 20 Jan 2025 17:44:09 +0200 Subject: [PATCH 100/416] Refactor executeScript to not load script on init, but on execute --- .../cache/RedisTbTransactionalCache.java | 21 +++++++------------ .../server/cache/VersionedRedisTbCache.java | 6 ------ 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index 97fc36fe27..379f08af37 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -271,24 +271,17 @@ public abstract class RedisTbTransactionalCache Date: Tue, 21 Jan 2025 10:03:50 +0200 Subject: [PATCH 101/416] fixed tests --- .../BaseCalculatedFieldConfiguration.java | 129 +----------------- .../CalculatedFieldConfiguration.java | 6 - .../ScriptCalculatedFieldConfiguration.java | 12 -- .../SimpleCalculatedFieldConfiguration.java | 12 -- .../dao/model/sql/CalculatedFieldEntity.java | 16 +-- ...efaultNativeCalculatedFieldRepository.java | 11 +- .../server/dao/service/AssetServiceTest.java | 7 +- .../service/CalculatedFieldServiceTest.java | 7 +- .../dao/service/CustomerServiceTest.java | 7 +- .../server/dao/service/DeviceServiceTest.java | 7 +- 10 files changed, 20 insertions(+), 194 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 8cab602683..87cece0419 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -15,48 +15,25 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.UUID; import java.util.stream.Collectors; @Data public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - @JsonIgnore - private final ObjectMapper mapper = new ObjectMapper(); - protected Map arguments; protected String expression; protected Output output; - public BaseCalculatedFieldConfiguration() { - } - - public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { -// BaseCalculatedFieldConfiguration calculatedFieldConfig = mapper.convertValue(config, BaseCalculatedFieldConfiguration.class); - BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); - this.arguments = calculatedFieldConfig.getArguments(); - this.expression = calculatedFieldConfig.getExpression(); - this.output = calculatedFieldConfig.getOutput(); - } - @Override public List getReferencedEntities() { return arguments.values().stream() @@ -70,7 +47,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.entrySet().stream() - .filter(entry -> entry.getValue().getRefEntityId().equals(entityId)) + .filter(entry -> entry.getValue().getRefEntityId() != null && entry.getValue().getRefEntityId().equals(entityId)) .forEach(entry -> { ReferencedEntityKey refEntityKey = entry.getValue().getRefEntityKey(); String argumentName = entry.getKey(); @@ -112,108 +89,4 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel return link; } - @Override - public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { - ObjectNode configNode = mapper.createObjectNode(); - - ObjectNode argumentsNode = configNode.putObject("arguments"); - arguments.forEach((key, argument) -> { - ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getRefEntityId(); - if (referencedEntityId != null) { - argumentNode.put("entityType", referencedEntityId.getEntityType().name()); - argumentNode.put("entityId", referencedEntityId.getId().toString()); - } else { - argumentNode.put("entityType", entityType.name()); - argumentNode.put("entityId", entityId.toString()); - } -// argumentNode.put("key", argument.getKey()); -// argumentNode.put("type", String.valueOf(argument.getType())); -// argumentNode.put("scope", String.valueOf(argument.getScope())); - argumentNode.put("defaultValue", argument.getDefaultValue()); - argumentNode.put("limit", String.valueOf(argument.getLimit())); - argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); - }); - - if (expression != null) { - configNode.put("expression", expression); - } - - if (output != null) { - ObjectNode outputNode = configNode.putObject("output"); - outputNode.put("name", output.getName()); - outputNode.put("type", String.valueOf(output.getType())); - if (output.getScope() != null) { - outputNode.put("scope", String.valueOf(output.getScope())); - } - } - - return configNode; - } - - private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { - if (config == null || !config.isObject()) { - return null; - } - - Map arguments = new HashMap<>(); - JsonNode argumentsNode = config.get("arguments"); - if (argumentsNode != null && argumentsNode.isObject()) { - argumentsNode.fields().forEachRemaining(entry -> { - String key = entry.getKey(); - JsonNode argumentNode = entry.getValue(); - Argument argument = new Argument(); - if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { - String referencedEntityType = argumentNode.get("entityType").asText(); - UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); - } else { - argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - } -// argument.setRefEntityKey(argumentNode.get("key").asText()); - JsonNode type = argumentNode.get("type"); -// if (type != null && !type.isNull() && !type.asText().equals("null")) { -// argument.setType(ArgumentType.valueOf(type.asText())); -// } -// JsonNode scope = argumentNode.get("scope"); -// if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { -// argument.setScope(AttributeScope.valueOf(scope.asText())); -// } - if (argumentNode.hasNonNull("defaultValue")) { - argument.setDefaultValue(argumentNode.get("defaultValue").asText()); - } - if (argumentNode.hasNonNull("limit")) { - argument.setLimit(argumentNode.get("limit").asInt()); - } - if (argumentNode.hasNonNull("timeWindow")) { - argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); - } - arguments.put(key, argument); - }); - } - this.setArguments(arguments); - - JsonNode expressionNode = config.get("expression"); - if (expressionNode != null && expressionNode.isTextual()) { - this.setExpression(expressionNode.asText()); - } - - JsonNode outputNode = config.get("output"); - if (outputNode != null) { - Output output = new Output(); - output.setName(outputNode.get("name").asText()); - JsonNode type = outputNode.get("type"); - if (type != null && !type.isNull() && !type.asText().equals("null")) { - output.setType(OutputType.valueOf(type.asText())); - } - JsonNode scope = outputNode.get("scope"); - if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { - output.setScope(AttributeScope.valueOf(scope.asText())); - } - this.setOutput(output); - } - - return this; - } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index ac94ade134..8f56bf491d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -18,8 +18,6 @@ package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.databind.JsonNode; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -29,7 +27,6 @@ import org.thingsboard.server.common.data.id.TenantId; import java.util.List; import java.util.Map; -import java.util.UUID; @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -57,9 +54,6 @@ public interface CalculatedFieldConfiguration { @JsonIgnore CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId); - @JsonIgnore - JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); - List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java index a24328b4c9..017fc5a485 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java @@ -15,24 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import java.util.UUID; - @Data public class ScriptCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - public ScriptCalculatedFieldConfiguration() { - super(); - } - - public ScriptCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java index af11d2f5d8..6312c3e1db 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -15,24 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import java.util.UUID; - @Data public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - public SimpleCalculatedFieldConfiguration() { - super(); - } - - public SimpleCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index d1352a26b8..a0157cde66 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -23,12 +23,9 @@ import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -96,8 +93,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); -// this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); - this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); + this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -113,8 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(CalculatedFieldType.valueOf(type)); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); -// calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); @@ -122,11 +117,4 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - return switch (CalculatedFieldType.valueOf(type)) { - case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - }; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index a5a2743f26..bb88982e3d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,8 +29,6 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -90,7 +88,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(type, configuration, entityType, entityId)); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); @@ -135,11 +133,4 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF }); } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { - return switch (type) { - case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - }; - } - } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index f38722c9c1..aed1621e1c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -885,9 +886,9 @@ public class AssetServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(savedAsset.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(savedAsset.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index a9a70d8bc8..6dd84714f8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -154,9 +155,9 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(referencedEntityId); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 236b159ddb..6e57279f38 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -380,9 +381,9 @@ public class CustomerServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(savedCustomer.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(savedCustomer.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index cca30e6fbc..959825e113 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1223,9 +1224,9 @@ public class DeviceServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(device.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(device.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); From d9ccc8118cf316389e3c284e468b4236118462bb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Jan 2025 10:52:06 +0200 Subject: [PATCH 102/416] fixed calculated field controller test --- .../server/controller/CalculatedFieldControllerTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 565321afbc..b1c7547251 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -141,9 +142,9 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(referencedEntityId); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); From 996b8997fdb6df1f2f3109a585b4866d0ca25be8 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 21 Jan 2025 11:14:22 +0200 Subject: [PATCH 103/416] Save time series strategies: simplify SQL upgrade script --- .../main/data/upgrade/basic/schema_update.sql | 78 +++++++------------ 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 6aaf8c3dbd..31833112ad 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -221,60 +221,36 @@ DO $$ WHERE table_name = 'rule_node' ) THEN - -- CREATE JSON validation function - CREATE OR REPLACE FUNCTION is_valid_jsonb(input text) - RETURNS boolean - LANGUAGE plpgsql - AS $func$ - DECLARE - dummy JSONB; - BEGIN - dummy := input::jsonb; - RETURN true; - EXCEPTION - WHEN others THEN - RETURN false; - END; - $func$; - UPDATE rule_node - SET configuration = CASE - -- Case 1: If configuration is NULL, invalid JSON, or not a JSON object - set default configuration - WHEN configuration IS NULL - OR NOT is_valid_jsonb(configuration) - OR jsonb_typeof(configuration::jsonb) <> 'object' - THEN jsonb_build_object( - 'defaultTTL', 0, - 'useServerTs', false, - 'persistenceSettings', jsonb_build_object('type', 'ON_EVERY_MESSAGE') - ) - -- Case 2: If a valid JSON object with persistenceSettings (rule node was already upgraded) - leave unchanged - WHEN configuration::jsonb ? 'persistenceSettings' - THEN configuration::jsonb - -- Case 3: If a valid JSON object without persistenceSettings and skipLatestPersistence = 'true' (string 'true' or boolean true) - set latest to SKIP - WHEN configuration::jsonb ->> 'skipLatestPersistence' = 'true' - THEN (configuration::jsonb - 'skipLatestPersistence') - || jsonb_build_object( - 'persistenceSettings', jsonb_build_object( - 'type', 'ADVANCED', - 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), - 'latest', jsonb_build_object('type', 'SKIP'), - 'webSockets', jsonb_build_object('type', 'ON_EVERY_MESSAGE') - ) - ) - -- Case 4: If a valid JSON object without persistenceSettings and skipLatestPersistence not 'true' (everything else) - set all to ON_EVERY_MESSAGE - ELSE (configuration::jsonb - 'skipLatestPersistence') - || jsonb_build_object( - 'persistenceSettings', jsonb_build_object( - 'type', 'ON_EVERY_MESSAGE' - ) - ) - END::text, + SET configuration = ( + (configuration::jsonb - 'skipLatestPersistence') + || jsonb_build_object( + 'persistenceSettings', jsonb_build_object( + 'type', 'ADVANCED', + 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), + 'latest', jsonb_build_object('type', 'SKIP'), + 'webSockets', jsonb_build_object('type', 'ON_EVERY_MESSAGE') + ) + ) + )::text, configuration_version = 1 - WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' AND configuration_version = 0; + WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' + AND configuration_version = 0 + AND configuration::jsonb ->> 'skipLatestPersistence' = 'true'; - -- Drop the helper function - DROP FUNCTION is_valid_jsonb(text); + UPDATE rule_node + SET configuration = ( + (configuration::jsonb - 'skipLatestPersistence') + || jsonb_build_object( + 'persistenceSettings', jsonb_build_object( + 'type', 'ON_EVERY_MESSAGE' + ) + ) + )::text, + configuration_version = 1 + WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' + AND configuration_version = 0 + AND (configuration::jsonb ->> 'skipLatestPersistence' != 'true' OR configuration::jsonb ->> 'skipLatestPersistence' IS NULL); END IF; END; From f419a6e438382e79eb759fb4a6a0b9ef7a14fdc6 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 21 Jan 2025 11:17:05 +0200 Subject: [PATCH 104/416] Save time series strategies: remove unnecessary check in Java upgrade script --- .../thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 422065f81f..7a32ee47c1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -193,9 +193,6 @@ public class TbMsgTimeseriesNode implements TbNode { boolean hasChanges = false; switch (fromVersion) { case 0: - if (oldConfiguration.has("persistenceSettings") && !oldConfiguration.has("skipLatestPersistence")) { - break; - } hasChanges = true; JsonNode skipLatestPersistence = oldConfiguration.get("skipLatestPersistence"); if (skipLatestPersistence != null && "true".equals(skipLatestPersistence.asText())) { From 39e47cd484c7e0afb6d642b88c3f388728c037f2 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 21 Jan 2025 14:26:08 +0200 Subject: [PATCH 105/416] Save time series strategies: refactor boolean flags in TimeseriesSaveRequest.java to SaveActions nested record --- .../DefaultTbEntityViewService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 19 ++++--- .../DefaultTbEntityViewServiceTest.java | 4 +- ...faultTelemetrySubscriptionServiceTest.java | 28 +++------- .../engine/api/TimeseriesSaveRequest.java | 33 +++++------- .../engine/api/TimeseriesSaveRequestTest.java | 26 +++++++-- .../engine/telemetry/TbMsgTimeseriesNode.java | 54 +++++++------------ .../rule/engine/math/TbMathNodeTest.java | 4 +- .../telemetry/TbMsgTimeseriesNodeTest.java | 42 ++++----------- 9 files changed, 86 insertions(+), 128 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java index ee807da750..3ff1387103 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java @@ -348,9 +348,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen .tenantId(entityView.getTenantId()) .entityId(entityId) .entries(latestValues) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) .callback(new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index dbd7967989..ab95ec6cba 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -118,10 +118,10 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer EntityId entityId = request.getEntityId(); checkInternalEntity(entityId); boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null; - if (sysTenant || !request.isSaveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { + if (sysTenant || !request.getSaveActions().saveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { KvUtils.validate(request.getEntries(), valueNoXssValidation); ListenableFuture future = saveTimeseriesInternal(request); - if (request.isSaveTimeseries()) { + if (request.getSaveActions().saveTimeseries()) { FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); Futures.addCallback(future, callback, tsCallBackExecutor); } @@ -134,22 +134,23 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); + TimeseriesSaveRequest.SaveActions saveActions = request.getSaveActions(); ListenableFuture saveFuture; - if (request.isSaveTimeseries() && request.isSaveLatest()) { + if (saveActions.saveTimeseries() && saveActions.saveLatest()) { saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); - } else if (request.isSaveLatest()) { + } else if (saveActions.saveLatest()) { saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor()); - } else if (request.isSaveTimeseries()) { + } else if (saveActions.saveTimeseries()) { saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } else { saveFuture = Futures.immediateFuture(0); } addMainCallback(saveFuture, request.getCallback()); - if (request.isSendWsUpdate()) { + if (saveActions.sendWsUpdate()) { addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); } - if (request.isSaveLatest()) { + if (saveActions.saveLatest()) { copyLatestToEntityViews(tenantId, entityId, request.getEntries()); } return saveFuture; @@ -236,9 +237,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .tenantId(tenantId) .entityId(entityView.getId()) .entries(entityViewLatest) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) .callback(new FutureCallback<>() { @Override public void onSuccess(@Nullable Void tmp) {} diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java index b36890f775..d6f803c1fe 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java @@ -93,9 +93,7 @@ class DefaultTbEntityViewServiceTest { .entityId(entityView.getId()) .entries(latest) .ttl(0L) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) .build(); var actualCopyLatestRequest = captor.getValue(); diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java index 8162ef4555..21d3aac607 100644 --- a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -173,9 +173,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(true) - .saveLatest(false) - .sendWsUpdate(false) + .saveActions(new TimeseriesSaveRequest.SaveActions(true, false, false)) .callback(emptyCallback) .build(); @@ -195,9 +193,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) .callback(emptyCallback) .build(); @@ -220,9 +216,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(true) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) .future(future) .build(); @@ -248,9 +242,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) .future(future) .build(); @@ -283,9 +275,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(false) + .saveActions(new TimeseriesSaveRequest.SaveActions(false, true, false)) .callback(emptyCallback) .build(); @@ -312,9 +302,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(true) - .saveLatest(false) - .sendWsUpdate(false) + .saveActions(new TimeseriesSaveRequest.SaveActions(true, false, false)) .callback(emptyCallback) .build(); @@ -340,9 +328,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(saveTimeseries) - .saveLatest(saveLatest) - .sendWsUpdate(sendWsUpdate) + .saveActions(new TimeseriesSaveRequest.SaveActions(saveTimeseries, saveLatest, sendWsUpdate)) .callback(emptyCallback) .build(); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 1ad0fd3408..d1122cc70d 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -38,11 +38,18 @@ public class TimeseriesSaveRequest { private final EntityId entityId; private final List entries; private final long ttl; - private final boolean saveTimeseries; - private final boolean saveLatest; - private final boolean sendWsUpdate; + private final SaveActions saveActions; private final FutureCallback callback; + public record SaveActions(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { + + public static final SaveActions SAVE_ALL = new SaveActions(true, true, true); + public static final SaveActions WS_ONLY = new SaveActions(false, false, true); + public static final SaveActions LATEST_AND_WS = new SaveActions(false, true, true); + public static final SaveActions SKIP_ALL = new SaveActions(false, false, false); + + } + public static Builder builder() { return new Builder(); } @@ -54,9 +61,7 @@ public class TimeseriesSaveRequest { private EntityId entityId; private List entries; private long ttl; - private boolean saveTimeseries = true; - private boolean saveLatest = true; - private boolean sendWsUpdate = true; + private SaveActions saveActions = SaveActions.SAVE_ALL; private FutureCallback callback; Builder() {} @@ -94,18 +99,8 @@ public class TimeseriesSaveRequest { return this; } - public Builder saveTimeseries(boolean saveTimeseries) { - this.saveTimeseries = saveTimeseries; - return this; - } - - public Builder saveLatest(boolean saveLatest) { - this.saveLatest = saveLatest; - return this; - } - - public Builder sendWsUpdate(boolean sendWsUpdate) { - this.sendWsUpdate = sendWsUpdate; + public Builder saveActions(SaveActions settings) { + this.saveActions = settings; return this; } @@ -129,7 +124,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveTimeseries, saveLatest, sendWsUpdate, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveActions, callback); } } diff --git a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java index cf05f9905b..38ce10a272 100644 --- a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java +++ b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java @@ -22,12 +22,30 @@ import static org.assertj.core.api.Assertions.assertThat; class TimeseriesSaveRequestTest { @Test - void testBooleanFlagsDefaultToTrue() { + void testDefaultSaveActionsAreSaveAll() { var request = TimeseriesSaveRequest.builder().build(); - assertThat(request.isSaveTimeseries()).isTrue(); - assertThat(request.isSaveLatest()).isTrue(); - assertThat(request.isSendWsUpdate()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + } + + @Test + void testSaveActionsSaveAll() { + assertThat(TimeseriesSaveRequest.SaveActions.SAVE_ALL).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, true, true)); + } + + @Test + void testSaveActionsWsOnly() { + assertThat(TimeseriesSaveRequest.SaveActions.WS_ONLY).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, false, true)); + } + + @Test + void testSaveActionsLatestAndWs() { + assertThat(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, true, true)); + } + + @Test + void testSaveActionsSkipAll() { + assertThat(TimeseriesSaveRequest.SaveActions.SKIP_ALL).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, false, false)); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 7a32ee47c1..658d635fae 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -107,13 +107,10 @@ public class TbMsgTimeseriesNode implements TbNode { } long ts = computeTs(msg, config.isUseServerTs()); - PersistenceDecision persistenceDecision = makePersistenceDecision(ts, msg.getOriginator().getId()); - boolean saveTimeseries = persistenceDecision.saveTimeseries(); - boolean saveLatest = persistenceDecision.saveLatest(); - boolean sendWsUpdate = persistenceDecision.sendWsUpdate(); + TimeseriesSaveRequest.SaveActions saveActions = determineSaveActions(ts, msg.getOriginator().getId()); // short-circuit - if (!saveTimeseries && !saveLatest && !sendWsUpdate) { + if (!saveActions.saveTimeseries() && !saveActions.saveLatest() && !saveActions.sendWsUpdate()) { ctx.tellSuccess(msg); return; } @@ -141,9 +138,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entityId(msg.getOriginator()) .entries(tsKvEntryList) .ttl(ttl) - .saveTimeseries(saveTimeseries) - .saveLatest(saveLatest) - .sendWsUpdate(sendWsUpdate) + .saveActions(saveActions) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } @@ -152,35 +147,26 @@ public class TbMsgTimeseriesNode implements TbNode { return ignoreMetadataTs ? System.currentTimeMillis() : msg.getMetaDataTs(); } - private record PersistenceDecision(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) {} - - private PersistenceDecision makePersistenceDecision(long ts, UUID originatorUuid) { - boolean saveTimeseries; - boolean saveLatest; - boolean sendWsUpdate; - + private TimeseriesSaveRequest.SaveActions determineSaveActions(long ts, UUID originatorUuid) { if (persistenceSettings instanceof OnEveryMessage) { - saveTimeseries = true; - saveLatest = true; - sendWsUpdate = true; - } else if (persistenceSettings instanceof WebSocketsOnly) { - saveTimeseries = false; - saveLatest = false; - sendWsUpdate = true; - } else if (persistenceSettings instanceof Deduplicate deduplicate) { + return TimeseriesSaveRequest.SaveActions.SAVE_ALL; + } + if (persistenceSettings instanceof WebSocketsOnly) { + return TimeseriesSaveRequest.SaveActions.WS_ONLY; + } + if (persistenceSettings instanceof Deduplicate deduplicate) { boolean isFirstMsgInInterval = deduplicate.getDeduplicateStrategy().shouldPersist(ts, originatorUuid); - saveTimeseries = isFirstMsgInInterval; - saveLatest = isFirstMsgInInterval; - sendWsUpdate = isFirstMsgInInterval; - } else if (persistenceSettings instanceof Advanced advanced) { - saveTimeseries = advanced.timeseries().shouldPersist(ts, originatorUuid); - saveLatest = advanced.latest().shouldPersist(ts, originatorUuid); - sendWsUpdate = advanced.webSockets().shouldPersist(ts, originatorUuid); - } else { // should not happen - throw new IllegalArgumentException("Unknown persistence settings type: " + persistenceSettings.getClass().getSimpleName()); + return isFirstMsgInInterval ? TimeseriesSaveRequest.SaveActions.SAVE_ALL : TimeseriesSaveRequest.SaveActions.SKIP_ALL; } - - return new PersistenceDecision(saveTimeseries, saveLatest, sendWsUpdate); + if (persistenceSettings instanceof Advanced advanced) { + return new TimeseriesSaveRequest.SaveActions( + advanced.timeseries().shouldPersist(ts, originatorUuid), + advanced.latest().shouldPersist(ts, originatorUuid), + advanced.webSockets().shouldPersist(ts, originatorUuid) + ); + } + // should not happen + throw new IllegalArgumentException("Unknown persistence settings type: " + persistenceSettings.getClass().getSimpleName()); } @Override diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java index 360fbce5b0..1270c8cf6c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java @@ -533,7 +533,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.isSaveLatest()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); @@ -569,7 +569,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.isSaveLatest()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index d0727b1a3c..15904ca1ff 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -207,7 +207,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).usingRecursiveFieldByFieldElementComparatorIgnoringFields("ts").containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(extractTtlAsSeconds(tenantProfile)); - assertThat(request.isSaveLatest()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -264,9 +264,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(config.getDefaultTTL()); - assertThat(request.isSaveTimeseries()).isTrue(); - assertThat(request.isSaveLatest()).isFalse(); - assertThat(request.isSendWsUpdate()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, false, true)); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -305,7 +303,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getCustomerId()).isNull(); assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getTtl()).isEqualTo(expectedTtl); - assertThat(request.isSaveLatest()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); } @@ -354,9 +352,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveTimeseries(true) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -391,9 +387,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveTimeseries(true) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -428,9 +422,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveTimeseries(false) - .saveLatest(false) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.WS_ONLY) .build(); node.onMsg(ctxMock, msg); @@ -469,9 +461,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveTimeseries(true) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -508,11 +498,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts1)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> { - assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); - assertThat(actualSaveRequest.isSaveLatest()).isTrue(); - assertThat(actualSaveRequest.isSendWsUpdate()).isTrue(); - } + actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL) )); clearInvocations(telemetryServiceMock); @@ -524,11 +510,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts2)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> { - assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); - assertThat(actualSaveRequest.isSaveLatest()).isFalse(); - assertThat(actualSaveRequest.isSendWsUpdate()).isFalse(); - } + actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, false, false)) )); clearInvocations(telemetryServiceMock); @@ -540,11 +522,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts3)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> { - assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); - assertThat(actualSaveRequest.isSaveLatest()).isTrue(); - assertThat(actualSaveRequest.isSendWsUpdate()).isFalse(); - } + actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, true, false)) )); } From 0f34f131c991012f9c499e36475064f0c4a7b97c Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 21 Jan 2025 14:30:27 +0200 Subject: [PATCH 106/416] Return TimeseriesSaveResult --- .../DefaultTelemetrySubscriptionService.java | 35 +++++----- .../telemetry/InternalTelemetryService.java | 3 +- .../dao/timeseries/TimeseriesService.java | 9 +-- .../common/data/kv/TimeseriesSaveResult.java | 26 +++++++ .../dao/timeseries/BaseTimeseriesService.java | 69 ++++++++----------- 5 files changed, 79 insertions(+), 63 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f873e48774..e7dfbcdca9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -126,10 +127,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null; if (sysTenant || request.isOnlyLatest() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { KvUtils.validate(request.getEntries(), valueNoXssValidation); - ListenableFuture future = saveTimeseriesInternal(request); + ListenableFuture future = saveTimeseriesInternal(request); if (!request.isOnlyLatest()) { - FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); - Futures.addCallback(future, callback, tsCallBackExecutor); + Futures.addCallback(future, getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant), tsCallBackExecutor); } } else { request.getCallback().onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); @@ -137,27 +137,27 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } @Override - public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { + public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - ListenableFuture saveFuture; + ListenableFuture resultFuture; if (request.isOnlyLatest()) { - saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor()); + resultFuture = tsService.saveLatest(tenantId, entityId, request.getEntries()); } else if (request.isSaveLatest()) { - saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); + resultFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); } else { - saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); + resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } - addMainCallback(saveFuture, request.getCallback()); - addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); + addMainCallback(resultFuture, request.getCallback()); + addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); - return saveFuture; + addCallback(resultFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + return resultFuture; } @Override @@ -329,19 +329,18 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant, FutureCallback callback) { + private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant) { return new FutureCallback<>() { @Override - public void onSuccess(Integer result) { - if (!sysTenant && result != null && result > 0) { - apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + public void onSuccess(TimeseriesSaveResult result) { + Integer dataPoints = result.getDataPoints(); + if (!sysTenant && dataPoints != null && dataPoints > 0) { + apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, dataPoints); } - callback.onSuccess(null); } @Override public void onFailure(Throwable t) { - callback.onFailure(t); } }; } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 8e45b84a75..8a236bf002 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -21,13 +21,14 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; /** * Created by ashvayka on 27.03.18. */ public interface InternalTelemetryService extends RuleEngineTelemetryService { - ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); + ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); void saveAttributesInternal(AttributesSaveRequest request); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 49918f3823..542f77445b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; @@ -44,13 +45,13 @@ public interface TimeseriesService { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); + ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries); ListenableFuture> remove(TenantId tenantId, EntityId entityId, List queries); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java new file mode 100644 index 0000000000..edec063be2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import lombok.Data; + +import java.util.List; + +@Data(staticConstructor = "of") +public class TimeseriesSaveResult { + private final Integer dataPoints; + private final List versions; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 756b73d88b..6b4a3c917e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.dao.entityview.EntityViewService; @@ -156,60 +157,48 @@ public class BaseTimeseriesService implements TimeseriesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { validate(entityId); - List> futures = new ArrayList<>(INSERTS_PER_ENTRY); - saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L); - return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); + return doSave(tenantId, entityId, List.of(tsKvEntry), 0L, true, true); } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - return doSave(tenantId, entityId, tsKvEntries, ttl, true); + public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + return doSave(tenantId, entityId, tsKvEntries, ttl, true, true); } @Override - public ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - return doSave(tenantId, entityId, tsKvEntries, ttl, false); - } - - private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest) { - int inserts = saveLatest ? INSERTS_PER_ENTRY : INSERTS_PER_ENTRY_WITHOUT_LATEST; - List> futures = new ArrayList<>(tsKvEntries.size() * inserts); - for (TsKvEntry tsKvEntry : tsKvEntries) { - if (saveLatest) { - saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); - } else { - saveWithoutLatestAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); - } - } - return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); + public ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + return doSave(tenantId, entityId, tsKvEntries, ttl, false, true); } @Override - public ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { - List> futures = new ArrayList<>(tsKvEntries.size()); - for (TsKvEntry tsKvEntry : tsKvEntries) { - futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); - } - return Futures.allAsList(futures); + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { + return doSave(tenantId, entityId, tsKvEntries, 0L, true, false); } - private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - futures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); - } - - private void saveWithoutLatestAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - } - - private void doSaveAndRegisterFuturesFor(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest, boolean saveTs) { + if (saveTs && entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only"); } - futures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey())); - futures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); + List> tsFutures = saveTs ? new ArrayList<>(tsKvEntries.size() * INSERTS_PER_ENTRY_WITHOUT_LATEST) : null; + List> latestFutures = saveLatest ? new ArrayList<>(tsKvEntries.size()) : null; + for (TsKvEntry tsKvEntry : tsKvEntries) { + if (saveTs) { + tsFutures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey())); + tsFutures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); + } + if (saveLatest) { + latestFutures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); + } + } + ListenableFuture dpsFuture = saveTs ? Futures.transform(Futures.allAsList(tsFutures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()) : Futures.immediateFuture(0); + ListenableFuture> versionsFuture = saveLatest ? Futures.allAsList(latestFutures) : Futures.immediateFuture(null); + return Futures.whenAllComplete(dpsFuture, versionsFuture).call(() -> { + Integer dataPoints = Futures.getUnchecked(dpsFuture); + List versions = Futures.getUnchecked(versionsFuture); + return TimeseriesSaveResult.of(dataPoints, versions); + }, MoreExecutors.directExecutor()); } private List updateQueriesForEntityView(EntityView entityView, List queries) { From 148521eddf6bdd95b44ab0986f83f7b8895656d3 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 21 Jan 2025 16:10:35 +0200 Subject: [PATCH 107/416] Save time series strategies: use @NotNull annotation instead of manual check --- .../rule/engine/telemetry/TbMsgTimeseriesNode.java | 3 --- .../telemetry/TbMsgTimeseriesNodeConfiguration.java | 2 ++ .../engine/telemetry/TbMsgTimeseriesNodeTest.java | 11 ++++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 658d635fae..eb2862a0da 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -89,9 +89,6 @@ public class TbMsgTimeseriesNode implements TbNode { ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); persistenceSettings = config.getPersistenceSettings(); - if (persistenceSettings == null) { - throw new TbNodeException("Persistence settings cannot be null", true); - } } private void onTenantProfileUpdate(TenantProfile tenantProfile) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java index f702ecc6da..fe22e42710 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.Getter; import org.thingsboard.rule.engine.api.NodeConfiguration; @@ -37,6 +38,7 @@ public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) - .isInstanceOf(TbNodeException.class) - .matches(e -> ((TbNodeException) e).isUnrecoverable()) - .hasMessage("Persistence settings cannot be null"); + assertThatThrownBy(() -> ConstraintValidator.validateFields(config)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Validation error: persistenceSettings must not be null"); } @ParameterizedTest From e009967fa74e58a0da5fd883d4a452f0303fba95 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 21 Jan 2025 16:21:28 +0200 Subject: [PATCH 108/416] Save time series strategies: rename SaveActions to Strategy --- .../DefaultTbEntityViewService.java | 2 +- .../DefaultTelemetrySubscriptionService.java | 18 ++++++++--------- .../DefaultTbEntityViewServiceTest.java | 2 +- ...faultTelemetrySubscriptionServiceTest.java | 14 ++++++------- .../engine/api/TimeseriesSaveRequest.java | 20 +++++++++---------- .../engine/api/TimeseriesSaveRequestTest.java | 20 +++++++++---------- .../engine/telemetry/TbMsgTimeseriesNode.java | 16 +++++++-------- .../rule/engine/math/TbMathNodeTest.java | 4 ++-- .../telemetry/TbMsgTimeseriesNodeTest.java | 20 +++++++++---------- 9 files changed, 58 insertions(+), 58 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java index 3ff1387103..2c384aa493 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java @@ -348,7 +348,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen .tenantId(entityView.getTenantId()) .entityId(entityId) .entries(latestValues) - .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) + .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .callback(new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ab95ec6cba..cab2f18dc7 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -118,10 +118,10 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer EntityId entityId = request.getEntityId(); checkInternalEntity(entityId); boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null; - if (sysTenant || !request.getSaveActions().saveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { + if (sysTenant || !request.getStrategy().saveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { KvUtils.validate(request.getEntries(), valueNoXssValidation); ListenableFuture future = saveTimeseriesInternal(request); - if (request.getSaveActions().saveTimeseries()) { + if (request.getStrategy().saveTimeseries()) { FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); Futures.addCallback(future, callback, tsCallBackExecutor); } @@ -134,23 +134,23 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - TimeseriesSaveRequest.SaveActions saveActions = request.getSaveActions(); + TimeseriesSaveRequest.Strategy strategy = request.getStrategy(); ListenableFuture saveFuture; - if (saveActions.saveTimeseries() && saveActions.saveLatest()) { + if (strategy.saveTimeseries() && strategy.saveLatest()) { saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); - } else if (saveActions.saveLatest()) { + } else if (strategy.saveLatest()) { saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor()); - } else if (saveActions.saveTimeseries()) { + } else if (strategy.saveTimeseries()) { saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } else { saveFuture = Futures.immediateFuture(0); } addMainCallback(saveFuture, request.getCallback()); - if (saveActions.sendWsUpdate()) { + if (strategy.sendWsUpdate()) { addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); } - if (saveActions.saveLatest()) { + if (strategy.saveLatest()) { copyLatestToEntityViews(tenantId, entityId, request.getEntries()); } return saveFuture; @@ -237,7 +237,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .tenantId(tenantId) .entityId(entityView.getId()) .entries(entityViewLatest) - .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) + .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .callback(new FutureCallback<>() { @Override public void onSuccess(@Nullable Void tmp) {} diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java index d6f803c1fe..aa6bfde935 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java @@ -93,7 +93,7 @@ class DefaultTbEntityViewServiceTest { .entityId(entityView.getId()) .entries(latest) .ttl(0L) - .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) + .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .build(); var actualCopyLatestRequest = captor.getValue(); diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java index 21d3aac607..10fdd85504 100644 --- a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -173,7 +173,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(new TimeseriesSaveRequest.SaveActions(true, false, false)) + .strategy(new TimeseriesSaveRequest.Strategy(true, false, false)) .callback(emptyCallback) .build(); @@ -193,7 +193,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) + .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .callback(emptyCallback) .build(); @@ -216,7 +216,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) .future(future) .build(); @@ -242,7 +242,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) + .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .future(future) .build(); @@ -275,7 +275,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(new TimeseriesSaveRequest.SaveActions(false, true, false)) + .strategy(new TimeseriesSaveRequest.Strategy(false, true, false)) .callback(emptyCallback) .build(); @@ -302,7 +302,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(new TimeseriesSaveRequest.SaveActions(true, false, false)) + .strategy(new TimeseriesSaveRequest.Strategy(true, false, false)) .callback(emptyCallback) .build(); @@ -328,7 +328,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(new TimeseriesSaveRequest.SaveActions(saveTimeseries, saveLatest, sendWsUpdate)) + .strategy(new TimeseriesSaveRequest.Strategy(saveTimeseries, saveLatest, sendWsUpdate)) .callback(emptyCallback) .build(); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index d1122cc70d..fb667fbfb2 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -38,15 +38,15 @@ public class TimeseriesSaveRequest { private final EntityId entityId; private final List entries; private final long ttl; - private final SaveActions saveActions; + private final Strategy strategy; private final FutureCallback callback; - public record SaveActions(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { + public record Strategy(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { - public static final SaveActions SAVE_ALL = new SaveActions(true, true, true); - public static final SaveActions WS_ONLY = new SaveActions(false, false, true); - public static final SaveActions LATEST_AND_WS = new SaveActions(false, true, true); - public static final SaveActions SKIP_ALL = new SaveActions(false, false, false); + public static final Strategy SAVE_ALL = new Strategy(true, true, true); + public static final Strategy WS_ONLY = new Strategy(false, false, true); + public static final Strategy LATEST_AND_WS = new Strategy(false, true, true); + public static final Strategy SKIP_ALL = new Strategy(false, false, false); } @@ -61,7 +61,7 @@ public class TimeseriesSaveRequest { private EntityId entityId; private List entries; private long ttl; - private SaveActions saveActions = SaveActions.SAVE_ALL; + private Strategy strategy = Strategy.SAVE_ALL; private FutureCallback callback; Builder() {} @@ -99,8 +99,8 @@ public class TimeseriesSaveRequest { return this; } - public Builder saveActions(SaveActions settings) { - this.saveActions = settings; + public Builder strategy(Strategy strategy) { + this.strategy = strategy; return this; } @@ -124,7 +124,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveActions, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, strategy, callback); } } diff --git a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java index 38ce10a272..321892991e 100644 --- a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java +++ b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java @@ -22,30 +22,30 @@ import static org.assertj.core.api.Assertions.assertThat; class TimeseriesSaveRequestTest { @Test - void testDefaultSaveActionsAreSaveAll() { + void testDefaultSaveStrategyIsSaveAll() { var request = TimeseriesSaveRequest.builder().build(); - assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); } @Test - void testSaveActionsSaveAll() { - assertThat(TimeseriesSaveRequest.SaveActions.SAVE_ALL).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, true, true)); + void testSaveAllStrategy() { + assertThat(TimeseriesSaveRequest.Strategy.SAVE_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(true, true, true)); } @Test - void testSaveActionsWsOnly() { - assertThat(TimeseriesSaveRequest.SaveActions.WS_ONLY).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, false, true)); + void testWsOnlyStrategy() { + assertThat(TimeseriesSaveRequest.Strategy.WS_ONLY).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, true)); } @Test - void testSaveActionsLatestAndWs() { - assertThat(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, true, true)); + void testLatestAndWsStrategy() { + assertThat(TimeseriesSaveRequest.Strategy.LATEST_AND_WS).isEqualTo(new TimeseriesSaveRequest.Strategy(false, true, true)); } @Test - void testSaveActionsSkipAll() { - assertThat(TimeseriesSaveRequest.SaveActions.SKIP_ALL).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, false, false)); + void testSkipAllStrategy() { + assertThat(TimeseriesSaveRequest.Strategy.SKIP_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, false)); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index eb2862a0da..53261bf7dd 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -104,10 +104,10 @@ public class TbMsgTimeseriesNode implements TbNode { } long ts = computeTs(msg, config.isUseServerTs()); - TimeseriesSaveRequest.SaveActions saveActions = determineSaveActions(ts, msg.getOriginator().getId()); + TimeseriesSaveRequest.Strategy strategy = determineSaveActions(ts, msg.getOriginator().getId()); // short-circuit - if (!saveActions.saveTimeseries() && !saveActions.saveLatest() && !saveActions.sendWsUpdate()) { + if (!strategy.saveTimeseries() && !strategy.saveLatest() && !strategy.sendWsUpdate()) { ctx.tellSuccess(msg); return; } @@ -135,7 +135,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entityId(msg.getOriginator()) .entries(tsKvEntryList) .ttl(ttl) - .saveActions(saveActions) + .strategy(strategy) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } @@ -144,19 +144,19 @@ public class TbMsgTimeseriesNode implements TbNode { return ignoreMetadataTs ? System.currentTimeMillis() : msg.getMetaDataTs(); } - private TimeseriesSaveRequest.SaveActions determineSaveActions(long ts, UUID originatorUuid) { + private TimeseriesSaveRequest.Strategy determineSaveActions(long ts, UUID originatorUuid) { if (persistenceSettings instanceof OnEveryMessage) { - return TimeseriesSaveRequest.SaveActions.SAVE_ALL; + return TimeseriesSaveRequest.Strategy.SAVE_ALL; } if (persistenceSettings instanceof WebSocketsOnly) { - return TimeseriesSaveRequest.SaveActions.WS_ONLY; + return TimeseriesSaveRequest.Strategy.WS_ONLY; } if (persistenceSettings instanceof Deduplicate deduplicate) { boolean isFirstMsgInInterval = deduplicate.getDeduplicateStrategy().shouldPersist(ts, originatorUuid); - return isFirstMsgInInterval ? TimeseriesSaveRequest.SaveActions.SAVE_ALL : TimeseriesSaveRequest.SaveActions.SKIP_ALL; + return isFirstMsgInInterval ? TimeseriesSaveRequest.Strategy.SAVE_ALL : TimeseriesSaveRequest.Strategy.SKIP_ALL; } if (persistenceSettings instanceof Advanced advanced) { - return new TimeseriesSaveRequest.SaveActions( + return new TimeseriesSaveRequest.Strategy( advanced.timeseries().shouldPersist(ts, originatorUuid), advanced.latest().shouldPersist(ts, originatorUuid), advanced.webSockets().shouldPersist(ts, originatorUuid) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java index 1270c8cf6c..63432afc37 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java @@ -533,7 +533,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); @@ -569,7 +569,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index 71236c97be..02c19ed5fc 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -208,7 +208,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).usingRecursiveFieldByFieldElementComparatorIgnoringFields("ts").containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(extractTtlAsSeconds(tenantProfile)); - assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -265,7 +265,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(config.getDefaultTTL()); - assertThat(request.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, false, true)); + assertThat(request.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, false, true)); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -304,7 +304,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getCustomerId()).isNull(); assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getTtl()).isEqualTo(expectedTtl); - assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); } @@ -353,7 +353,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -388,7 +388,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -423,7 +423,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveActions(TimeseriesSaveRequest.SaveActions.WS_ONLY) + .strategy(TimeseriesSaveRequest.Strategy.WS_ONLY) .build(); node.onMsg(ctxMock, msg); @@ -462,7 +462,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -499,7 +499,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts1)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL) )); clearInvocations(telemetryServiceMock); @@ -511,7 +511,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts2)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, false, false)) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, false, false)) )); clearInvocations(telemetryServiceMock); @@ -523,7 +523,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts3)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, true, false)) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, true, false)) )); } From 9ef68584c9aecb19659496210cf914a7eb5f81b5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Jan 2025 09:40:12 +0200 Subject: [PATCH 109/416] updated getMappedTelemetry method --- ...efaultCalculatedFieldExecutionService.java | 19 ++++++++++--------- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- ...CalculatedFieldAttributeUpdateRequest.java | 15 ++------------- ...CalculatedFieldTelemetryUpdateRequest.java | 5 +---- ...alculatedFieldTimeSeriesUpdateRequest.java | 13 +++---------- 5 files changed, 17 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7c17d54649..84a21ab315 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -271,7 +271,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !isProfileEntity(entry.getValue().getRefEntityId())) + .filter(entry -> entry.getValue().getRefEntityId() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { @@ -375,9 +375,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { if (cfTargetEntityId != null) { calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, cfTargetEntityId); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + EntityId targetEntityId = isProfileEntity(cfTargetEntityId) ? request.getEntityId() : cfTargetEntityId; + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } }); } @@ -406,9 +407,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); if (targetEntityTpi.isMyPartition()) { - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + executeTelemetryUpdate(ctx, targetEntity, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); @@ -427,13 +428,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } proto.getLinksList().forEach(ctxIdProto -> { - EntityId entityId = request.getEntityId(); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, entityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } }); } catch (Exception e) { @@ -654,7 +655,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { EntityId argumentEntityId = argument.getRefEntityId(); - EntityId entityId = isProfileEntity(argumentEntityId) + EntityId entityId = (argumentEntityId == null || isProfileEntity(argumentEntityId)) ? targetEntityId : argumentEntityId; return fetchKvEntry(tenantId, entityId, argument); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e17a1a61f9..cb4052b7df 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -57,7 +57,7 @@ public class CalculatedFieldCtx { this.arguments = configuration.getArguments(); this.referencedEntityKeys = arguments.entrySet().stream() .collect(Collectors.toMap( - entry -> new TbPair<>(entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), + entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), Map.Entry::getKey )); this.argNames = new ArrayList<>(arguments.keySet()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 6050370fd2..d2eb31cd6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -19,7 +19,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -52,16 +51,7 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel } @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { - return switch (scope) { - case CLIENT_SCOPE -> linkConfiguration.getClientAttributes(); - case SERVER_SCOPE -> linkConfiguration.getServerAttributes(); - case SHARED_SCOPE -> linkConfiguration.getSharedAttributes(); - }; - } - - @Override - public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + public Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId) { Map mappedKvEntries = new HashMap<>(); Map, String> referencedKeys = ctx.getReferencedEntityKeys(); @@ -70,7 +60,7 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - String argName = referencedKeys.get(new TbPair<>(entityId, referencedEntityKey)); + String argName = referencedKeys.get(new TbPair<>(referencedEntityId, referencedEntityKey)); if (argName != null) { mappedKvEntries.put(argName, entry); @@ -79,5 +69,4 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel return mappedKvEntries; } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index f85117dc41..3f7250f4ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,8 +34,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { List getPreviousCalculatedFieldIds(); - Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); - - Map getMappedTelemetry(CalculatedFieldCtx ctx); + Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 646145a46e..a5637c8cfd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf.telemetry; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -49,12 +48,7 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe } @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { - return linkConfiguration.getTimeSeries(); - } - - @Override - public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + public Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId) { Map mappedKvEntries = new HashMap<>(); Map, String> referencedKeys = ctx.getReferencedEntityKeys(); @@ -62,13 +56,13 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe String key = entry.getKey(); ReferencedEntityKey tsLatestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - String argTsLatestName = referencedKeys.get(new TbPair<>(entityId, tsLatestKey)); + String argTsLatestName = referencedKeys.get(new TbPair<>(referencedEntityId, tsLatestKey)); if (argTsLatestName != null) { mappedKvEntries.put(argTsLatestName, entry); } else { ReferencedEntityKey tsRollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); - String argTsRollingName = referencedKeys.get(new TbPair<>(entityId, tsRollingKey)); + String argTsRollingName = referencedKeys.get(new TbPair<>(referencedEntityId, tsRollingKey)); if (argTsRollingName != null) { mappedKvEntries.put(argTsRollingName, entry); @@ -78,5 +72,4 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe return mappedKvEntries; } - } From a2095636a07af68c083e4fe3bb06e92089082589 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Wed, 22 Jan 2025 10:56:36 +0200 Subject: [PATCH 110/416] Save time series strategies: add max deduplication interval validation --- .../strategy/DeduplicatePersistenceStrategy.java | 6 ++++-- .../strategy/DeduplicatePersistenceStrategyTest.java | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java index e4eb982861..513523f8f3 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java @@ -28,14 +28,16 @@ import java.util.UUID; final class DeduplicatePersistenceStrategy implements PersistenceStrategy { private static final int MIN_DEDUPLICATION_INTERVAL_SECS = 1; + private static final int MAX_DEDUPLICATION_INTERVAL_SECS = (int) Duration.ofDays(1L).toSeconds(); private final long deduplicationIntervalMillis; private final LoadingCache> deduplicationCache; @JsonCreator public DeduplicatePersistenceStrategy(@JsonProperty("deduplicationIntervalSecs") int deduplicationIntervalSecs) { - if (deduplicationIntervalSecs < MIN_DEDUPLICATION_INTERVAL_SECS) { - throw new IllegalArgumentException("Deduplication interval must be at least " + MIN_DEDUPLICATION_INTERVAL_SECS + " second(s), was " + deduplicationIntervalSecs + " second(s)"); + if (deduplicationIntervalSecs < MIN_DEDUPLICATION_INTERVAL_SECS || deduplicationIntervalSecs > MAX_DEDUPLICATION_INTERVAL_SECS) { + throw new IllegalArgumentException("Deduplication interval must be at least " + MIN_DEDUPLICATION_INTERVAL_SECS + " second(s) " + + "and at most " + MAX_DEDUPLICATION_INTERVAL_SECS + " second(s), was " + deduplicationIntervalSecs + " second(s)"); } deduplicationIntervalMillis = Duration.ofSeconds(deduplicationIntervalSecs).toMillis(); deduplicationCache = Caffeine.newBuilder() diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java index 3f57a7f3df..8aeda400b4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java @@ -39,7 +39,14 @@ class DeduplicatePersistenceStrategyTest { void shouldThrowWhenDeduplicationIntervalIsLessThanOneSecond() { assertThatThrownBy(() -> new DeduplicatePersistenceStrategy(0)) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Deduplication interval must be at least 1 second(s), was 0 second(s)"); + .hasMessageContaining("Deduplication interval must be at least 1 second(s) and at most 86400 second(s), was 0 second(s)"); + } + + @Test + void shouldThrowWhenDeduplicationIntervalIsMoreThan24Hours() { + assertThatThrownBy(() -> new DeduplicatePersistenceStrategy(86401)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Deduplication interval must be at least 1 second(s) and at most 86400 second(s), was 86401 second(s)"); } @Test From 349554f93870ac6b4e34bfa1a0657f8d56759d92 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Wed, 22 Jan 2025 12:17:26 +0200 Subject: [PATCH 111/416] Save time series strategies: dynamically calculate max number of deduplication intervals --- .../engine/telemetry/TbMsgTimeseriesNode.java | 4 +- .../DeduplicatePersistenceStrategy.java | 14 ++++- .../DeduplicatePersistenceStrategyTest.java | 55 +++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 53261bf7dd..232b8a3d36 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -104,7 +104,7 @@ public class TbMsgTimeseriesNode implements TbNode { } long ts = computeTs(msg, config.isUseServerTs()); - TimeseriesSaveRequest.Strategy strategy = determineSaveActions(ts, msg.getOriginator().getId()); + TimeseriesSaveRequest.Strategy strategy = determineSaveStrategy(ts, msg.getOriginator().getId()); // short-circuit if (!strategy.saveTimeseries() && !strategy.saveLatest() && !strategy.sendWsUpdate()) { @@ -144,7 +144,7 @@ public class TbMsgTimeseriesNode implements TbNode { return ignoreMetadataTs ? System.currentTimeMillis() : msg.getMetaDataTs(); } - private TimeseriesSaveRequest.Strategy determineSaveActions(long ts, UUID originatorUuid) { + private TimeseriesSaveRequest.Strategy determineSaveStrategy(long ts, UUID originatorUuid) { if (persistenceSettings instanceof OnEveryMessage) { return TimeseriesSaveRequest.Strategy.SAVE_ALL; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java index 513523f8f3..601328c304 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java @@ -30,6 +30,9 @@ final class DeduplicatePersistenceStrategy implements PersistenceStrategy { private static final int MIN_DEDUPLICATION_INTERVAL_SECS = 1; private static final int MAX_DEDUPLICATION_INTERVAL_SECS = (int) Duration.ofDays(1L).toSeconds(); + private static final int MAX_TOTAL_INTERVALS_DURATION_SECS = (int) Duration.ofDays(2L).toSeconds(); + private static final int MAX_NUMBER_OF_INTERVALS = 100; + private final long deduplicationIntervalMillis; private final LoadingCache> deduplicationCache; @@ -43,10 +46,19 @@ final class DeduplicatePersistenceStrategy implements PersistenceStrategy { deduplicationCache = Caffeine.newBuilder() .softValues() .expireAfterAccess(Duration.ofSeconds(deduplicationIntervalSecs * 10L)) - .maximumSize(20L) + .maximumSize(calculateMaxNumberOfDeduplicationIntervals(deduplicationIntervalSecs)) .build(__ -> Sets.newConcurrentHashSet()); } + /** + * Calculates the maximum number of deduplication intervals we will store in the cache. + * We limit retention to two days to avoid stale data and cap it at 100 intervals to manage memory usage. + */ + private static long calculateMaxNumberOfDeduplicationIntervals(int deduplicationIntervalSecs) { + int numberOfDeduplicationIntervals = MAX_TOTAL_INTERVALS_DURATION_SECS / deduplicationIntervalSecs; + return Math.min(numberOfDeduplicationIntervals, MAX_NUMBER_OF_INTERVALS); + } + @JsonProperty("deduplicationIntervalSecs") public long getDeduplicationIntervalSecs() { return Duration.ofMillis(deduplicationIntervalMillis).toSeconds(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java index 8aeda400b4..1a6bdc831d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java @@ -15,10 +15,14 @@ */ package org.thingsboard.rule.engine.telemetry.strategy; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.Policy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; import java.time.Duration; +import java.util.Set; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -49,6 +53,57 @@ class DeduplicatePersistenceStrategyTest { .hasMessageContaining("Deduplication interval must be at least 1 second(s) and at most 86400 second(s), was 86401 second(s)"); } + @Test + void shouldNotAllowMoreThan100DeduplicationIntervals() { + // GIVEN + int deduplicationIntervalSecs = 1; // min deduplication interval duration + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().eviction()) + .isPresent() + .map(Policy.Eviction::getMaximum) + .hasValue(100L); + } + + @Test + void shouldCalculateMaxIntervalsAsTwoDaysDividedByIntervalDuration() { + // GIVEN + int deduplicationIntervalSecs = (int) Duration.ofHours(1L).toSeconds(); + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().eviction()) + .isPresent() + .map(Policy.Eviction::getMaximum) + .hasValue(48L); + } + + @Test + void shouldKeepAtLeastTwoDeduplicationIntervals() { + // GIVEN + int deduplicationIntervalSecs = (int) Duration.ofDays(1L).toSeconds(); // max deduplication interval duration + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().eviction()) + .isPresent() + .map(Policy.Eviction::getMaximum) + .hasValue(2L); + } + @Test void shouldReturnTrueForFirstCallInInterval() { long ts = 1_000_000L; From 6b9d374a5f2957d14a3722c2e6a6459da211db11 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Jan 2025 12:23:42 +0200 Subject: [PATCH 112/416] Tmp commit for merge --- .../cf/CalculatedFieldExecutionService.java | 11 +++ .../TbCalculatedFieldConsumerService.java | 8 ++ .../DefaultTelemetrySubscriptionService.java | 18 +++-- .../src/main/resources/thingsboard.yml | 12 ++- .../server/common/data/DataConstants.java | 2 + .../server/common/msg/queue/ServiceType.java | 3 +- common/proto/src/main/proto/queue.proto | 73 +++++++------------ .../queue/discovery/HashPartitionService.java | 12 ++- .../discovery/event/PartitionChangeEvent.java | 4 + 9 files changed, 86 insertions(+), 57 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 8ba1f6dfed..e18c8b4119 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -21,6 +21,17 @@ import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdat public interface CalculatedFieldExecutionService { + /** + * Push incoming telemetry to the CF processing queue for async processing. + * @param request - telemetry request; + * @param callback - callback to be executed when the message is ack by the queue. + */ + void pushRequestToQueue(CalculatedFieldTelemetryUpdateRequest request, TbCallback callback); + + void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); + + /* ===================================================== */ + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java new file mode 100644 index 0000000000..387bdd7143 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -0,0 +1,8 @@ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +public interface TbCalculatedFieldConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 8773564e5d..dcb72b8dd0 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.telemetry; +import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -128,8 +129,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer KvUtils.validate(request.getEntries(), valueNoXssValidation); ListenableFuture future = saveTimeseriesInternal(request); if (!request.isOnlyLatest()) { - FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); - Futures.addCallback(future, callback, tsCallBackExecutor); + Futures.addCallback(future, getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant), tsCallBackExecutor); } } else { request.getCallback().onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); @@ -148,7 +148,14 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } else { saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } - + // We need to guarantee, that the message is successfully pushed to the calculated fields service before we execute any callbacks. + saveFuture = Futures.transformAsync(saveFuture, new AsyncFunction() { + @Override + public ListenableFuture apply(Integer input) throws Exception { + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)); + return input; + } + }); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { @@ -326,19 +333,18 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant, FutureCallback callback) { + private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant) { return new FutureCallback<>() { @Override public void onSuccess(Integer result) { if (!sysTenant && result != null && result > 0) { apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); } - callback.onSuccess(null); } @Override public void onFailure(Throwable t) { - callback.onFailure(t); + } }; } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 3014e1448b..5151bc019b 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1692,7 +1692,6 @@ queue: enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" # Statistics printing interval for Housekeeper print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" - vc: # Default topic name topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" @@ -1739,6 +1738,17 @@ queue: topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" + calculated-fields: + # Topic name for Calculated Field (CF) tasks + topic: "${TB_QUEUE_CF_TOPIC:tb_calculated_fields}" + # Interval in milliseconds to poll messages by CF (Rule Engine) microservices + poll-interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" + # Amount of partitions used by CF microservices + partitions: "${TB_QUEUE_CF_PARTITIONS:10}" + # Timeout for processing a message pack by CF microservices + pack-processing-timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" + # Enable/disable a separate consumer per partition for CF queue + consumer-per-partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 56a5e135f8..77a7c4a781 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -145,4 +145,6 @@ public class DataConstants { public static final String EDGE_QUEUE_NAME = "Edge"; public static final String EDGE_EVENT_QUEUE_NAME = "EdgeEvent"; + public static final String CF_QUEUE_NAME = "CalculatedFields"; + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index f3a0e47d09..022c46bcc6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -26,7 +26,8 @@ public enum ServiceType { TB_RULE_ENGINE("TB Rule Engine"), TB_TRANSPORT("TB Transport"), JS_EXECUTOR("JS Executor"), - TB_VC_EXECUTOR("TB VC Executor"); + TB_VC_EXECUTOR("TB VC Executor"), + TB_CF_ENGINE("TB Calculated Fields Engine"); private final String label; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1036d5ba67..4e719a02df 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -183,18 +183,6 @@ message TsKvListProto { repeated KeyValueProto kv = 2; } -message AttributeKvProto { - AttributeKey key = 1; - AttributeValueProto value = 2; -} - -message TelemetryProto { - oneof proto { - AttributeKvProto attrKv = 1; - TsKvProto tsKv = 2; - } -} - message DeviceInfoProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -785,17 +773,7 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } -message CalculatedFieldMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} - -message EntityProfileUpdateMsgProto { +message CalculatedFieldEntityUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; string entityType = 3; @@ -806,31 +784,26 @@ message EntityProfileUpdateMsgProto { int64 oldProfileIdLSB = 8; int64 newProfileIdMSB = 9; int64 newProfileIdLSB = 10; + bool added = 11; + bool updated = 12; + bool deleted = 13; } -message ProfileEntityMsgProto { +message CalculatedFieldTelemetryMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; string entityType = 3; int64 entityIdMSB = 4; int64 entityIdLSB = 5; - string entityProfileType = 6; - int64 profileIdMSB = 7; - int64 profileIdLSB = 8; - bool added = 9; - bool deleted = 10; + repeated CalculatedFieldIdProto previousCalculatedFields = 7; + repeated TsKvProto tsData = 9; + AttributeScopeProto scope = 10; + repeated AttributeValueProto attrData = 11; } -message TelemetryUpdateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - string entityType = 3; - int64 entityIdMSB = 4; - int64 entityIdLSB = 5; - repeated CalculatedFieldEntityCtxIdProto links = 6; - repeated CalculatedFieldIdProto previousCalculatedFields = 7; - string scope = 8; - repeated TelemetryProto updatedTelemetry = 9; +message CalculatedFieldLinkedTelemetryMsgProto { + CalculatedFieldTelemetryMsgProto msg = 1; + repeated CalculatedFieldEntityCtxIdProto links = 2; } message CalculatedFieldEntityCtxIdProto { @@ -1589,9 +1562,8 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; - CalculatedFieldMsgProto calculatedFieldMsg = 53; - EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; - ProfileEntityMsgProto profileEntityMsg = 55; +// CalculatedFieldMsgProto calculatedFieldMsg = 53; +// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1611,8 +1583,8 @@ message ToCoreNotificationMsg { FromEdgeSyncResponseMsgProto fromEdgeSyncResponse = 12 [deprecated = true]; ResourceCacheInvalidateMsg resourceCacheInvalidateMsg = 13; RestApiCallResponseMsgProto restApiCallResponseMsg = 50; - EntityProfileUpdateMsgProto entityProfileUpdateMsg = 51; - ProfileEntityMsgProto profileEntityMsg = 52; +// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 51; +// ProfileEntityMsgProto profileEntityMsg = 52; } /* Messages to Edge queue that are handled by ThingsBoard Core Service */ @@ -1632,6 +1604,16 @@ message ToEdgeEventNotificationMsg { EdgeEventMsgProto edgeEventMsg = 1; } +message ToCalculatedFieldMsg { + CalculatedFieldTelemetryMsgProto telemetryMsg = 1; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; +} + +message ToCalculatedFieldNotificationMsg { + ComponentLifecycleMsgProto componentLifecycle = 1; + CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 2; +} + /* Messages that are handled by ThingsBoard RuleEngine Service */ message ToRuleEngineMsg { int64 tenantIdMSB = 1; @@ -1639,9 +1621,6 @@ message ToRuleEngineMsg { bytes tbMsg = 3; repeated string relationTypes = 4; string failureMessage = 5; - TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; - EntityProfileUpdateMsgProto entityProfileUpdateMsg = 7; - ProfileEntityMsgProto profileEntityMsg = 8; } message ToRuleEngineNotificationMsg { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 37e519e3f2..53bdc78c93 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -51,8 +51,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.DataConstants.EDGE_QUEUE_NAME; -import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.*; @Service @Slf4j @@ -62,6 +61,10 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:10}") private Integer corePartitions; + @Value("${queue.calculated-fields.topic}") + private String cfTopic; + @Value("${queue.calculated-fields.partitions:10}") + private Integer cfPartitions; @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @Value("${queue.vc.partitions:10}") @@ -108,10 +111,15 @@ public class HashPartitionService implements PartitionService { @PostConstruct public void init() { this.hashFunction = forName(hashFunctionName); + QueueKey coreKey = new QueueKey(ServiceType.TB_CORE); partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); + QueueKey cfKey = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + partitionSizesMap.put(cfKey, cfPartitions); + partitionTopicsMap.put(cfKey, cfTopic); + QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); partitionTopicsMap.put(vcKey, vcTopic); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 88ceb4aa08..3bb0c56f9a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -52,6 +52,10 @@ public class PartitionChangeEvent extends TbApplicationEvent { return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_CORE, DataConstants.EDGE_QUEUE_NAME); } + public Set getCalculatedFieldsPartitions() { + return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); + } + private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { return partitionsMap.entrySet() .stream() From 2c8c272051e0228e143fbaa74d8f656ddf762c79 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 22 Jan 2025 12:51:49 +0100 Subject: [PATCH 113/416] fixed NPE in sendRpcResponseToTbCore --- .../server/service/rpc/DefaultTbRuleEngineRpcService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java index debc1cb987..27a64dc443 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -115,8 +115,9 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), src.getTenantId(), src.getDeviceId(), src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody()), src.isPersisted(), src.getRetries(), src.getAdditionalInfo()); forwardRpcRequestToDeviceActor(request, response -> { - if (src.isRestApiCall()) { - sendRpcResponseToTbCore(src.getOriginServiceId(), response); + String originServiceId = src.getOriginServiceId(); + if (src.isRestApiCall() && originServiceId != null) { + sendRpcResponseToTbCore(originServiceId, response); } consumer.accept(RuleEngineDeviceRpcResponse.builder() .deviceId(src.getDeviceId()) From 0131358d3ba3b5e37c203c9500f7fc7ef477335e Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Wed, 22 Jan 2025 15:10:56 +0200 Subject: [PATCH 114/416] Save time series strategies: dynamically calculate interval expire after access --- .../DeduplicatePersistenceStrategy.java | 18 ++++++- .../DeduplicatePersistenceStrategyTest.java | 51 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java index 601328c304..868e0d8ca4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.collect.Sets; +import com.google.common.primitives.Longs; import java.time.Duration; import java.util.Set; @@ -30,6 +31,10 @@ final class DeduplicatePersistenceStrategy implements PersistenceStrategy { private static final int MIN_DEDUPLICATION_INTERVAL_SECS = 1; private static final int MAX_DEDUPLICATION_INTERVAL_SECS = (int) Duration.ofDays(1L).toSeconds(); + private static final long MIN_INTERVAL_EXPIRY_MILLIS = Duration.ofMinutes(10L).toMillis(); + private static final int INTERVAL_EXPIRY_FACTOR = 10; + private static final long MAX_INTERVAL_EXPIRY_MILLIS = Duration.ofDays(2L).toMillis(); + private static final int MAX_TOTAL_INTERVALS_DURATION_SECS = (int) Duration.ofDays(2L).toSeconds(); private static final int MAX_NUMBER_OF_INTERVALS = 100; @@ -45,11 +50,22 @@ final class DeduplicatePersistenceStrategy implements PersistenceStrategy { deduplicationIntervalMillis = Duration.ofSeconds(deduplicationIntervalSecs).toMillis(); deduplicationCache = Caffeine.newBuilder() .softValues() - .expireAfterAccess(Duration.ofSeconds(deduplicationIntervalSecs * 10L)) + .expireAfterAccess(calculateExpireAfterAccess(deduplicationIntervalSecs)) .maximumSize(calculateMaxNumberOfDeduplicationIntervals(deduplicationIntervalSecs)) .build(__ -> Sets.newConcurrentHashSet()); } + /** + * Calculates the expire-after-access duration. By default, we keep each deduplication interval + * alive for 10 “iterations” (interval duration × 10). However, we never let this drop below + * 10 minutes to ensure adequate retention for small intervals, nor exceed 48 hours to prevent + * storing stale data in memory. + */ + private static Duration calculateExpireAfterAccess(int deduplicationIntervalSecs) { + long desiredExpiryMillis = Duration.ofSeconds(deduplicationIntervalSecs).toMillis() * INTERVAL_EXPIRY_FACTOR; + return Duration.ofMillis(Longs.constrainToRange(desiredExpiryMillis, MIN_INTERVAL_EXPIRY_MILLIS, MAX_INTERVAL_EXPIRY_MILLIS)); + } + /** * Calculates the maximum number of deduplication intervals we will store in the cache. * We limit retention to two days to avoid stale data and cap it at 100 intervals to manage memory usage. diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java index 1a6bdc831d..1ae2b367a5 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java @@ -53,6 +53,57 @@ class DeduplicatePersistenceStrategyTest { .hasMessageContaining("Deduplication interval must be at least 1 second(s) and at most 86400 second(s), was 86401 second(s)"); } + @Test + void shouldUseAtLeastTenMinutesForExpireAfterAccess() { + // GIVEN + int deduplicationIntervalSecs = 1; // min deduplication interval duration + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().expireAfterAccess()) + .isPresent() + .map(Policy.FixedExpiration::getExpiresAfter) + .hasValue(Duration.ofMinutes(10L)); + } + + @Test + void shouldCalculateExpireAfterAccessAsIntervalDurationMultipliedByTen() { + // GIVEN + int deduplicationIntervalSecs = (int) Duration.ofHours(1L).toSeconds(); // max deduplication interval duration + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().expireAfterAccess()) + .isPresent() + .map(Policy.FixedExpiration::getExpiresAfter) + .hasValue(Duration.ofHours(10L)); + } + + @Test + void shouldUseAtMostTwoDaysForExpireAfterAccess() { + // GIVEN + int deduplicationIntervalSecs = (int) Duration.ofDays(1L).toSeconds(); // max deduplication interval duration + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().expireAfterAccess()) + .isPresent() + .map(Policy.FixedExpiration::getExpiresAfter) + .hasValue(Duration.ofDays(2L)); + } + @Test void shouldNotAllowMoreThan100DeduplicationIntervals() { // GIVEN From 2d1ead5f544520c4a35c7149e19399f181dd4d65 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 22 Jan 2025 15:43:59 +0200 Subject: [PATCH 115/416] UI: Add persistence settings in save ts rule node --- .../action/action-rule-node-config.module.ts | 10 +- ...ced-persistence-setting-row.component.html | 39 ++++++ ...anced-persistence-setting-row.component.ts | 114 ++++++++++++++++++ ...dvanced-persistence-setting.component.html | 31 +++++ .../advanced-persistence-setting.component.ts | 83 +++++++++++++ .../action/timeseries-config.component.html | 40 ++++++ .../action/timeseries-config.component.ts | 104 ++++++++++++++-- .../action/timeseries-config.models.ts | 78 ++++++++++++ .../common/time-unit-input.component.html | 3 +- .../assets/locale/locale.constant-en_US.json | 19 +++ 10 files changed, 511 insertions(+), 10 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/action-rule-node-config.module.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/action-rule-node-config.module.ts index 17307cb341..caec009629 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/action-rule-node-config.module.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/action-rule-node-config.module.ts @@ -42,6 +42,12 @@ import { DeleteAttributesConfigComponent } from './delete-attributes-config.comp import { MathFunctionConfigComponent } from './math-function-config.component'; import { DeviceStateConfigComponent } from './device-state-config.component'; import { SendRestApiCallReplyConfigComponent } from './send-rest-api-call-reply-config.component'; +import { + AdvancedPersistenceSettingComponent +} from '@home/components/rule-node/action/advanced-persistence-setting.component'; +import { + AdvancedPersistenceSettingRowComponent +} from '@home/components/rule-node/action/advanced-persistence-setting-row.component'; @NgModule({ declarations: [ @@ -67,7 +73,9 @@ import { SendRestApiCallReplyConfigComponent } from './send-rest-api-call-reply- PushToEdgeConfigComponent, PushToCloudConfigComponent, MathFunctionConfigComponent, - DeviceStateConfigComponent + DeviceStateConfigComponent, + AdvancedPersistenceSettingComponent, + AdvancedPersistenceSettingRowComponent, ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html new file mode 100644 index 0000000000..038fe41bb2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html @@ -0,0 +1,39 @@ + +
+
{{ title }}
+ + rule-node-config.save-time-series.strategy + + @for (strategy of persistenceStrategies; track strategy) { + {{ PersistenceTypeTranslationMap.get(strategy) | translate }} + } + + + @if(persistenceSettingRowForm.get('type').value === PersistenceType.DEDUPLICATE) { + + + } +
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts new file mode 100644 index 0000000000..64ea045f8f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts @@ -0,0 +1,114 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator +} from '@angular/forms'; +import { + AdvancedPersistenceConfig, + defaultAdvancedPersistenceConfig, + maxDeduplicateTimeSecs, + PersistenceType, + PersistenceTypeTranslationMap +} from '@home/components/rule-node/action/timeseries-config.models'; +import { isDefinedAndNotNull } from '@core/utils'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; + +@Component({ + selector: 'tb-advanced-persistence-setting-row', + templateUrl: './advanced-persistence-setting-row.component.html', + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => AdvancedPersistenceSettingRowComponent), + multi: true + },{ + provide: NG_VALIDATORS, + useExisting: forwardRef(() => AdvancedPersistenceSettingRowComponent), + multi: true + }] +}) +export class AdvancedPersistenceSettingRowComponent implements ControlValueAccessor, Validator { + + @Input() + title: string; + + persistenceSettingRowForm = this.fb.group({ + type: [defaultAdvancedPersistenceConfig.type], + deduplicationIntervalSecs: [{value: 60, disabled: true}] + }); + + PersistenceType = PersistenceType; + persistenceStrategies = [PersistenceType.ON_EVERY_MESSAGE, PersistenceType.DEDUPLICATE, PersistenceType.SKIP]; + PersistenceTypeTranslationMap = PersistenceTypeTranslationMap; + + maxDeduplicateTime = maxDeduplicateTimeSecs; + + private propagateChange: (value: any) => void = () => {}; + + constructor(private fb: FormBuilder) { + this.persistenceSettingRowForm.get('type').valueChanges.pipe( + takeUntilDestroyed() + ).subscribe(() => this.updatedValidation()); + + this.persistenceSettingRowForm.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((value) => this.propagateChange(value)); + } + + registerOnChange(fn: any) { + this.propagateChange = fn; + } + + registerOnTouched(_fn: any) { + } + + setDisabledState(isDisabled: boolean) { + if (isDisabled) { + this.persistenceSettingRowForm.disable({emitEvent: false}); + } else { + this.persistenceSettingRowForm.enable({emitEvent: false}); + this.updatedValidation(); + } + } + + validate(): ValidationErrors | null { + return this.persistenceSettingRowForm.valid ? null : { + persistenceSettingRow: false + }; + } + + writeValue(value: AdvancedPersistenceConfig) { + if (isDefinedAndNotNull(value)) { + this.persistenceSettingRowForm.patchValue(value, {emitEvent: false}); + } else { + this.persistenceSettingRowForm.patchValue(defaultAdvancedPersistenceConfig); + } + } + + private updatedValidation() { + if (this.persistenceSettingRowForm.get('type').value === PersistenceType.DEDUPLICATE) { + this.persistenceSettingRowForm.get('deduplicationIntervalSecs').enable({emitEvent: false}); + } else { + this.persistenceSettingRowForm.get('deduplicationIntervalSecs').disable({emitEvent: false}) + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html new file mode 100644 index 0000000000..eb0f05bec2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html @@ -0,0 +1,31 @@ + +
+ + + +
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts new file mode 100644 index 0000000000..5c9930fbb7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts @@ -0,0 +1,83 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator +} from '@angular/forms'; +import { Component, forwardRef } from '@angular/core'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { AdvancedPersistenceStrategy } from '@home/components/rule-node/action/timeseries-config.models'; + +@Component({ + selector: 'tb-advanced-persistence-settings', + templateUrl: './advanced-persistence-setting.component.html', + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => AdvancedPersistenceSettingComponent), + multi: true + },{ + provide: NG_VALIDATORS, + useExisting: forwardRef(() => AdvancedPersistenceSettingComponent), + multi: true + }] +}) +export class AdvancedPersistenceSettingComponent implements ControlValueAccessor, Validator { + + persistenceForm = this.fb.group({ + timeseries: [null], + latest: [null], + webSockets: [null] + }); + + private propagateChange: (value: any) => void = () => {}; + + constructor(private fb: FormBuilder) { + this.persistenceForm.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe(value => this.propagateChange(value)); + } + + registerOnChange(fn: any) { + this.propagateChange = fn; + } + + registerOnTouched(_fn: any) { + } + + setDisabledState(isDisabled: boolean) { + if (isDisabled) { + this.persistenceForm.disable({emitEvent: false}); + } else { + this.persistenceForm.enable({emitEvent: false}); + } + } + + validate(): ValidationErrors | null { + return this.persistenceForm.valid ? null : { + persistenceForm: false + }; + } + + writeValue(value: AdvancedPersistenceStrategy) { + this.persistenceForm.patchValue(value, {emitEvent: false}); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html index aa70914fa9..47655c1285 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html @@ -16,6 +16,46 @@ -->
+
+
+
+ rule-node-config.save-time-series.persistence-settings +
+ + {{ 'rule-node-config.basic-mode' | translate}} + {{ 'rule-node-config.advanced-mode' | translate }} + +
+ @if(!timeseriesConfigForm.get('persistenceSettings.isAdvanced').value) { + + rule-node-config.save-time-series.strategy + + @for (strategy of persistenceStrategies; track strategy) { + {{ PersistenceTypeTranslationMap.get(strategy) | translate }} + } + + + + @if(timeseriesConfigForm.get('persistenceSettings.type').value === PersistenceType.DEDUPLICATE) { + + + } + } + @else { + + } +
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts index 0c5283f54a..a834d0d43f 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts @@ -15,8 +15,18 @@ /// import { Component } from '@angular/core'; -import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; -import { RuleNodeConfiguration, RuleNodeConfigurationComponent } from '@shared/models/rule-node.models'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { RuleNodeConfigurationComponent } from '@shared/models/rule-node.models'; +import { + defaultAdvancedPersistenceStrategy, + maxDeduplicateTimeSecs, + PersistenceSettings, + PersistenceSettingsForm, + PersistenceType, + PersistenceTypeTranslationMap, + TimeseriesNodeConfiguration, + TimeseriesNodeConfigurationForm +} from '@home/components/rule-node/action/timeseries-config.models'; @Component({ selector: 'tb-action-node-timeseries-config', @@ -25,20 +35,98 @@ import { RuleNodeConfiguration, RuleNodeConfigurationComponent } from '@shared/m }) export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { - timeseriesConfigForm: UntypedFormGroup; + timeseriesConfigForm: FormGroup; - constructor(private fb: UntypedFormBuilder) { + PersistenceType = PersistenceType; + persistenceStrategies = [PersistenceType.ON_EVERY_MESSAGE, PersistenceType.DEDUPLICATE, PersistenceType.WEBSOCKETS_ONLY]; + PersistenceTypeTranslationMap = PersistenceTypeTranslationMap; + + maxDeduplicateTime = maxDeduplicateTimeSecs + + constructor(private fb: FormBuilder) { super(); } - protected configForm(): UntypedFormGroup { + protected configForm(): FormGroup { return this.timeseriesConfigForm; } - protected onConfigurationSet(configuration: RuleNodeConfiguration) { + protected validatorTriggers(): string[] { + return ['persistenceSettings.isAdvanced', 'persistenceSettings.type']; + } + + protected prepareInputConfig(config: TimeseriesNodeConfiguration): TimeseriesNodeConfigurationForm { + let persistenceSettings: PersistenceSettingsForm; + if (config?.persistenceSettings) { + const isAdvanced = config?.persistenceSettings?.type === PersistenceType.ADVANCED; + persistenceSettings = { + ...config.persistenceSettings, + isAdvanced: isAdvanced, + type: isAdvanced ? PersistenceType.ON_EVERY_MESSAGE : config.persistenceSettings.type, + advanced: isAdvanced ? config.persistenceSettings : defaultAdvancedPersistenceStrategy + } + } else { + persistenceSettings = { + type: PersistenceType.ON_EVERY_MESSAGE, + isAdvanced: false, + deduplicationIntervalSecs: 10, + advanced: defaultAdvancedPersistenceStrategy + }; + } + return { + ...config, + persistenceSettings: persistenceSettings + } + } + + protected prepareOutputConfig(config: TimeseriesNodeConfigurationForm): TimeseriesNodeConfiguration { + let persistenceSettings: PersistenceSettings; + if (config.persistenceSettings.isAdvanced) { + persistenceSettings = { + ...config.persistenceSettings.advanced, + type: PersistenceType.ADVANCED + }; + } else { + persistenceSettings = { + type: config.persistenceSettings.type, + deduplicationIntervalSecs: config.persistenceSettings?.deduplicationIntervalSecs + }; + } + return { + ...config, + persistenceSettings + }; + } + + protected onConfigurationSet(config: TimeseriesNodeConfigurationForm) { this.timeseriesConfigForm = this.fb.group({ - defaultTTL: [configuration ? configuration.defaultTTL : null, [Validators.required, Validators.min(0)]], - useServerTs: [configuration ? configuration.useServerTs : false, []] + persistenceSettings: this.fb.group({ + isAdvanced: [config?.persistenceSettings?.isAdvanced ?? false], + type: [config?.persistenceSettings?.type ?? PersistenceType.ON_EVERY_MESSAGE], + deduplicationIntervalSecs: [ + {value: config?.persistenceSettings?.deduplicationIntervalSecs ?? 10, disabled: true}, + [Validators.required, Validators.max(maxDeduplicateTimeSecs)] + ], + advanced: [{value: null, disabled: true}] + }), + defaultTTL: [config?.defaultTTL ?? null, [Validators.required, Validators.min(0)]], + useServerTs: [config?.useServerTs ?? false] }); } + + protected updateValidators(emitEvent: boolean, _trigger?: string) { + const persistenceForm = this.timeseriesConfigForm.get('persistenceSettings') as FormGroup; + const isAdvanced: boolean = persistenceForm.get('isAdvanced').value; + const type: PersistenceType = persistenceForm.get('type').value; + if (!isAdvanced && type === PersistenceType.DEDUPLICATE) { + persistenceForm.get('deduplicationIntervalSecs').enable({emitEvent}); + } else { + persistenceForm.get('deduplicationIntervalSecs').disable({emitEvent}); + } + if (isAdvanced) { + persistenceForm.get('advanced').enable({emitEvent}); + } else { + persistenceForm.get('advanced').disable({emitEvent}); + } + } } diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts new file mode 100644 index 0000000000..241787d511 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts @@ -0,0 +1,78 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { DAY, SECOND } from '@shared/models/time/time.models'; + +export const maxDeduplicateTimeSecs = DAY / SECOND + 1; + +export interface TimeseriesNodeConfiguration { + persistenceSettings: PersistenceSettings; + defaultTTL: number; + useServerTs: boolean; +} + +export interface TimeseriesNodeConfigurationForm extends Omit { + persistenceSettings: PersistenceSettingsForm +} + +export type PersistenceSettings = BasicPersistenceSettings & Partial & Partial; + +export type PersistenceSettingsForm = Omit & { + isAdvanced: boolean; + advanced?: Partial; + type: PersistenceType; +}; + +export enum PersistenceType { + ON_EVERY_MESSAGE = 'ON_EVERY_MESSAGE', + DEDUPLICATE = 'DEDUPLICATE', + WEBSOCKETS_ONLY = 'WEBSOCKETS_ONLY', + ADVANCED = 'ADVANCED', + SKIP = 'SKIP' +} + +export const PersistenceTypeTranslationMap = new Map([ + [PersistenceType.ON_EVERY_MESSAGE, 'rule-node-config.save-time-series.strategy-type.every-message'], + [PersistenceType.DEDUPLICATE, 'rule-node-config.save-time-series.strategy-type.deduplicate'], + [PersistenceType.WEBSOCKETS_ONLY, 'rule-node-config.save-time-series.strategy-type.web-sockets-only'], + [PersistenceType.SKIP, 'rule-node-config.save-time-series.strategy-type.skip'], +]) + +export interface BasicPersistenceSettings { + type: PersistenceType; +} + +export interface DeduplicatePersistenceStrategy extends BasicPersistenceSettings{ + deduplicationIntervalSecs: number; +} + +export interface AdvancedPersistenceStrategy extends BasicPersistenceSettings{ + timeseries: AdvancedPersistenceConfig; + latest: AdvancedPersistenceConfig; + webSockets: AdvancedPersistenceConfig; +} + +export type AdvancedPersistenceConfig = WithOptional; + +export const defaultAdvancedPersistenceConfig: AdvancedPersistenceConfig = { + type: PersistenceType.ON_EVERY_MESSAGE +} + +export const defaultAdvancedPersistenceStrategy: Omit = { + timeseries: defaultAdvancedPersistenceConfig, + latest: defaultAdvancedPersistenceConfig, + webSockets: defaultAdvancedPersistenceConfig, +} diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html index 79caab4cb7..1fd0a60159 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html @@ -16,12 +16,13 @@ -->
- + {{ labelText }}
+ {{ requiredText }} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index b6aa720a53..f19336d6ad 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5068,6 +5068,25 @@ "units": "Units", "tell-failure-aws-lambda": "Tell Failure if AWS Lambda function execution raises exception", "tell-failure-aws-lambda-hint": "Rule node forces failure of message processing if AWS Lambda function execution raises exception.", + "basic-mode": "Basic", + "advanced-mode": "Advanced", + "save-time-series": { + "persistence-settings": "Persistence settings", + "persistence-settings-hint": "Persistence settings hint", + "strategy": "Strategy", + "deduplication-interval": "Deduplication interval", + "deduplication-interval-required": "Deduplication interval is required", + "deduplication-interval-min-max-range": "Deduplication interval should be at least 1 second and at most 1 day", + "strategy-type": { + "every-message": "On every message", + "skip": "Skip", + "deduplicate": "Deduplicate", + "web-sockets-only": "WebSockets only" + }, + "time-series": "Time series", + "latest": "Latest", + "web-sockets": "WebSockets" + }, "key-val": { "key": "Key", "value": "Value", From 3dfcc318e070675df6eb95e2470c4b76fe132516 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Wed, 22 Jan 2025 16:21:47 +0200 Subject: [PATCH 116/416] Save time series strategies: add information about strategies in node description; refactor node description --- .../engine/telemetry/TbMsgTimeseriesNode.java | 55 +++++++++++++++---- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 232b8a3d36..31deddf63b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -57,18 +57,49 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE type = ComponentType.ACTION, name = "save time series", configClazz = TbMsgTimeseriesNodeConfiguration.class, - nodeDescription = "Saves time series data", - nodeDetails = "Saves time series telemetry data based on configurable TTL parameter. Expects messages with 'POST_TELEMETRY_REQUEST' message type. " + - "Timestamp in milliseconds will be taken from metadata.ts, otherwise 'now' message timestamp will be applied. " + - "Allows stopping updating values for incoming keys in the latest ts_kv table if 'skipLatestPersistence' is set to true.\n " + - "
" + - "Enable 'useServerTs' param to use the timestamp of the message processing instead of the timestamp from the message. " + - "Useful for all sorts of sequential processing if you merge messages from multiple sources (devices, assets, etc).\n" + - "
" + - "In the case of sequential processing, the platform guarantees that the messages are processed in the order of their submission to the queue. " + - "However, the timestamp of the messages originated by multiple devices/servers may be unsynchronized long before they are pushed to the queue. " + - "The DB layer has certain optimizations to ignore the updates of the \"attributes\" and \"latest values\" tables if the new record has a timestamp that is older than the previous record. " + - "So, to make sure that all the messages will be processed correctly, one should enable this parameter for sequential message processing scenarios.", + nodeDescription = """ + Saves time series data with a configurable TTL and according to configured persistence strategies. + """, + nodeDetails = """ + Node performs three actions: +
    +
  • Time series: save time series data to a ts_kv table in a DB.
    • +
  • Latest values: save time series data to a ts_kv_latest table in a DB.
    • +
  • WebSockets: notify WebSockets subscriptions about time series data updates.
    • +
+ + For each action, three persistence strategies are available: +
    +
  • On every message: perform the action for every message.
    • +
  • Deduplicate: perform the action only for the first message from a particular originator within a configurable interval.
    • +
  • Skip: never perform the action.
    • +
+ + Persistence strategies are configured using persistence settings, which support two modes: +
    +
  • Basic +
      +
    • On every message: applies the "On every message" strategy to all actions.
      • +
    • Deduplicate: applies the "Deduplicate" strategy (with a specified interval) to all actions.
      • +
    • WebSockets only: applies the "Skip" strategy to Time series and Latest values, and the "On every message" strategy to WebSockets.
      • +
    +
    • +
  • Advanced: configure each action’s strategy independently.
    • +
+ + By default, the timestamp is taken from metadata.ts. You can enable + Use server timestamp to always use the current server time instead. This is particularly + useful in sequential processing scenarios where messages may arrive with out-of-order timestamps from + multiple sources. Note that the DB layer may ignore older records for attributes and latest values, + so enabling Use server timestamp can ensure correct ordering. +

+ The TTL is taken first from metadata.TTL. If absent, the node configuration’s default + TTL is used. If neither is set, the tenant profile default applies. +

+ This node expects messages of type POST_TELEMETRY_REQUEST. +

+ Output connections: Success, Failure. + """, uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbActionNodeTimeseriesConfig", icon = "file_upload", From c047d5f4f065d17ea80935bb4f3ebb9eae518c80 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Jan 2025 17:01:28 +0200 Subject: [PATCH 117/416] added debug event entity --- .../server/actors/ActorSystemContext.java | 41 ++++++ .../actors/ruleChain/DefaultTbContext.java | 2 +- ...efaultCalculatedFieldExecutionService.java | 13 +- .../common/data/cf/CalculatedField.java | 23 ++- .../data/event/CalculatedFieldDebugEvent.java | 95 ++++++++++++ .../CalculatedFieldDebugEventFilter.java | 50 +++++++ .../server/common/data/event/EventType.java | 3 +- .../dao/cf/BaseCalculatedFieldService.java | 1 + .../server/dao/event/BaseEventService.java | 6 + .../server/dao/model/ModelConstants.java | 5 + .../sql/CalculatedFieldDebugEventEntity.java | 104 ++++++++++++++ .../dao/model/sql/CalculatedFieldEntity.java | 7 + .../CalculatedFieldDebugEventRepository.java | 135 ++++++++++++++++++ .../dao/sql/event/DedicatedJpaEventDao.java | 5 +- .../dao/sql/event/EventInsertRepository.java | 29 ++++ .../server/dao/sql/event/JpaBaseEventDao.java | 47 ++++++ .../main/resources/sql/schema-entities.sql | 17 +++ 17 files changed, 570 insertions(+), 13 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 71fa8fa958..e6f541090c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -42,10 +42,12 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.event.RuleChainDebugEvent; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -125,6 +127,7 @@ import org.thingsboard.server.service.transport.TbCoreToTransportService; import java.io.PrintWriter; import java.io.StringWriter; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; @@ -157,6 +160,18 @@ public class ActorSystemContext { } }; + private static final FutureCallback CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Void event) { + + } + + @Override + public void onFailure(Throwable th) { + log.error("Could not save debug Event for Calculated Field", th); + } + }; + private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); public ConcurrentMap getDebugPerTenantLimits() { @@ -723,6 +738,32 @@ public class ActorSystemContext { } } + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, TbMsg tbMsg, Throwable error) { + if (checkLimits(tenantId, tbMsg, error)) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder event = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(tbMsg.getOriginator()) + .msgId(tbMsg.getId()) + .msgType(tbMsg.getType()) + .arguments(JacksonUtil.toString(arguments)) + .result(tbMsg.getData()); + + if (error != null) { + event.error(toString(error)); + } + + ListenableFuture future = eventService.saveAsync(event.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } + } + } + private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) { if (debugPerTenantEnabled) { DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id -> diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 421e4efb26..54e32446bf 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; import lombok.extern.slf4j.Slf4j; import org.bouncycastle.util.Arrays; +import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; @@ -64,7 +65,6 @@ import org.thingsboard.server.common.data.msg.TbNodeConnectionType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.server.common.data.rule.RuleNodeState; import org.thingsboard.server.common.data.script.ScriptLanguage; import org.thingsboard.server.common.msg.TbActorMsg; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 84a21ab315..a365e2bb6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -42,7 +42,6 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -71,6 +70,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; @@ -122,6 +122,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; + private final EventService eventService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -253,7 +254,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - calculatedFieldCache.updateCalculatedField(tenantId, calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); if (!shouldReinit) { return; @@ -301,6 +301,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { + calculatedFieldCache.updateCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); callback.onSuccess(); shouldReinit = false; } @@ -329,13 +330,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); + boolean argumentsChanged = !oldCalculatedField.getConfiguration().getArguments().equals(newCalculatedField.getConfiguration().getArguments()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; + return entityIdChanged || typeChanged || argumentsChanged; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index e626c9d3d2..f4b92b3802 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSetter; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; @@ -22,11 +24,13 @@ import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasDebugSettings; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -36,7 +40,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @Schema @Data @EqualsAndHashCode(callSuper = true) -public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity { +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity, HasDebugSettings { private static final long serialVersionUID = 4491966747773381420L; @@ -50,6 +54,11 @@ public class CalculatedField extends BaseData implements HasN @Length(fieldName = "name") @Schema(description = "User defined name of the calculated field.") private String name; + @Deprecated + @Schema(description = "Enable/disable debug. ", example = "false", deprecated = true) + private boolean debugMode; + @Schema(description = "Debug settings object.") + private DebugSettings debugSettings; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; @Schema(implementation = SimpleCalculatedFieldConfiguration.class) @@ -109,4 +118,16 @@ public class CalculatedField extends BaseData implements HasN .toString(); } + // Getter is ignored for serialization + @JsonIgnore + public boolean isDebugMode() { + return debugMode; + } + + // Setter is annotated for deserialization + @JsonSetter + public void setDebugMode(boolean debugMode) { + this.debugMode = debugMode; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java new file mode 100644 index 0000000000..e0599db358 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class CalculatedFieldDebugEvent extends Event { + + private static final long serialVersionUID = -7091690784759639853L; + + @Builder + private CalculatedFieldDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, + CalculatedFieldId calculatedFieldId, EntityId eventEntity, UUID msgId, + String msgType, String arguments, String result, String error) { + super(tenantId, entityId, serviceId, id, ts); + this.calculatedFieldId = calculatedFieldId; + this.eventEntity = eventEntity; + this.msgId = msgId; + this.msgType = msgType; + this.arguments = arguments; + this.result = result; + this.error = error; + } + + @Getter + private final CalculatedFieldId calculatedFieldId; + @Getter + private final EntityId eventEntity; + @Getter + private final UUID msgId; + @Getter + private final String msgType; + @Getter + @Setter + private String arguments; + @Getter + @Setter + private String result; + @Getter + @Setter + private String error; + + @Override + public EventType getType() { + return EventType.DEBUG_CALCULATED_FIELD; + } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("calculatedFieldId", calculatedFieldId.toString()); + if (eventEntity != null) { + json.put("entityId", eventEntity.getId().toString()) + .put("entityType", eventEntity.getEntityType().name()); + } + if (msgId != null) { + json.put("msgId", msgId.toString()); + } + putNotNull(json, "msgType", msgType); + putNotNull(json, "arguments", arguments); + putNotNull(json, "result", result); + putNotNull(json, "error", error); + return eventInfo; + } + + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java new file mode 100644 index 0000000000..839583ef6c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.StringUtils; + +@Data +@EqualsAndHashCode(callSuper = true) +@Schema +public class CalculatedFieldDebugEventFilter extends DebugEventFilter { + + @Schema(description = "String value representing the calculated field id in the event body", example = "ccbfa2fe-c8f5-45d8-bb37-6b61a6e02833") + protected String calculatedFieldId; + @Schema(description = "String value representing the entity id in the event body", example = "57b6bafe-d600-423c-9267-fe31e5218986") + protected String entityId; + @Schema(description = "String value representing the entity type", allowableValues = "DEVICE") + protected String entityType; + @Schema(description = "String value representing the message id in the rule engine", example = "dcf44612-2ce4-4e5d-b462-ebb9c5628228") + protected String msgId; + @Schema(description = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") + protected String msgType; + + @Override + public EventType getEventType() { + return EventType.DEBUG_CALCULATED_FIELD; + } + + @Override + public boolean isNotEmpty() { + return super.isNotEmpty() || !StringUtils.isEmpty(calculatedFieldId) || !StringUtils.isEmpty(entityId) + || !StringUtils.isEmpty(entityType) || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java index 6f98e1537f..46d0e49370 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java @@ -22,7 +22,8 @@ public enum EventType { LC_EVENT("lc_event", "LC_EVENT"), STATS("stats_event", "STATS"), DEBUG_RULE_NODE("rule_node_debug_event", "DEBUG_RULE_NODE", true), - DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true); + DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true), + DEBUG_CALCULATED_FIELD("cf_debug_event", "DEBUG_CALCULATED_FIELD", true); @Getter private final String table; diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 9c81d91f64..26ed4134cc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -64,6 +64,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 5314dcd405..dc57b07f72 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; @@ -88,6 +89,11 @@ public class BaseEventService implements EventService { ErrorEvent eEvent = (ErrorEvent) event; truncateField(eEvent, ErrorEvent::getError, ErrorEvent::setError); break; + case DEBUG_CALCULATED_FIELD: + CalculatedFieldDebugEvent cfEvent = (CalculatedFieldDebugEvent) event; + truncateField(cfEvent, CalculatedFieldDebugEvent::getArguments, CalculatedFieldDebugEvent::setArguments); + truncateField(cfEvent, CalculatedFieldDebugEvent::getResult, CalculatedFieldDebugEvent::setResult); + truncateField(cfEvent, CalculatedFieldDebugEvent::getError, CalculatedFieldDebugEvent::setError); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 6bd6a3a384..0cc2f03d2e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -373,6 +373,7 @@ public class ModelConstants { public static final String STATS_EVENT_TABLE_NAME = "stats_event"; public static final String RULE_NODE_DEBUG_EVENT_TABLE_NAME = "rule_node_debug_event"; public static final String RULE_CHAIN_DEBUG_EVENT_TABLE_NAME = "rule_chain_debug_event"; + public static final String CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME = "cf_debug_event"; public static final String EVENT_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String EVENT_SERVICE_ID_PROPERTY = "service_id"; @@ -397,6 +398,10 @@ public class ModelConstants { public static final String EVENT_METADATA_COLUMN_NAME = "e_metadata"; public static final String EVENT_MESSAGE_COLUMN_NAME = "e_message"; + public static final String EVENT_CALCULATED_FIELD_ID_COLUMN_NAME = "cf_id"; + public static final String EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME = "e_args"; + public static final String EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME = "e_result"; + public static final String DEBUG_MODE = "debug_mode"; public static final String DEBUG_SETTINGS = "debug_settings"; public static final String SINGLETON_MODE = "singleton_mode"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java new file mode 100644 index 0000000000..57849ae3ea --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_TYPE_COLUMN_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME) +@NoArgsConstructor +public class CalculatedFieldDebugEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_CALCULATED_FIELD_ID_COLUMN_NAME) + private UUID calculatedFieldId; + @Column(name = EVENT_ENTITY_ID_COLUMN_NAME) + private UUID eventEntityId; + @Column(name = EVENT_ENTITY_TYPE_COLUMN_NAME) + private String eventEntityType; + @Column(name = EVENT_MSG_ID_COLUMN_NAME) + private UUID msgId; + @Column(name = EVENT_MSG_TYPE_COLUMN_NAME) + private String msgType; + @Column(name = EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME) + private String arguments; + @Column(name = EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME) + private String result; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public CalculatedFieldDebugEventEntity(CalculatedFieldDebugEvent event) { + super(event); + if (event.getCalculatedFieldId() != null) { + this.calculatedFieldId = event.getCalculatedFieldId().getId(); + } + if (event.getEventEntity() != null) { + this.eventEntityId = event.getEventEntity().getId(); + this.eventEntityType = event.getEventEntity().getEntityType().name(); + } + this.msgId = event.getMsgId(); + this.msgType = event.getMsgType(); + this.arguments = event.getArguments(); + this.result = event.getResult(); + this.error = event.getError(); + } + + @Override + public CalculatedFieldDebugEvent toData() { + var builder = CalculatedFieldDebugEvent.builder() + .id(id) + .tenantId(TenantId.fromUUID(tenantId)) + .ts(ts) + .serviceId(serviceId) + .entityId(entityId) + .msgId(msgId) + .msgType(msgType) + .arguments(arguments) + .result(result) + .error(error); + if (calculatedFieldId != null) { + builder.calculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + } + if (eventEntityId != null) { + builder.eventEntity(EntityIdFactory.getByTypeAndUuid(eventEntityType, eventEntityId)); + } + return builder.build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index a0157cde66..64c8e8d5b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -26,6 +26,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -45,6 +46,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_T import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TENANT_ID_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TYPE; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_VERSION; +import static org.thingsboard.server.dao.model.ModelConstants.DEBUG_SETTINGS; @Data @EqualsAndHashCode(callSuper = true) @@ -77,6 +79,9 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem @Column(name = CALCULATED_FIELD_VERSION) private Long version; + @Column(name = DEBUG_SETTINGS) + private String debugSettings; + @Column(name = CALCULATED_FIELD_EXTERNAL_ID) private UUID externalId; @@ -95,6 +100,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.configurationVersion = calculatedField.getConfigurationVersion(); this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.version = calculatedField.getVersion(); + this.debugSettings = JacksonUtil.toString(calculatedField.getDebugSettings()); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); } @@ -111,6 +117,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); + calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java new file mode 100644 index 0000000000..c0bd21ca74 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java @@ -0,0 +1,135 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.dao.model.sql.CalculatedFieldDebugEventEntity; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldDebugEventRepository extends EventRepository, JpaRepository { + + @Override + @Query(nativeQuery = true, value = "SELECT * FROM cf_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); + + @Override + @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM rule_node_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("calculatedFieldId") UUID calculatedFieldId, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("isError") boolean isError, + @Param("error") String error, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM CalculatedFieldDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))") + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("calculatedFieldId") UUID calculatedFieldId, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("isError") boolean isError, + @Param("error") String error); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java index 9b7af5e7f7..7bf9b426e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java @@ -36,10 +36,11 @@ public class DedicatedJpaEventDao extends JpaBaseEventDao { RuleNodeDebugEventRepository ruleNodeDebugEventRepository, RuleChainDebugEventRepository ruleChainDebugEventRepository, ScheduledLogExecutorComponent logExecutor, - StatsFactory statsFactory) { + StatsFactory statsFactory, + CalculatedFieldDebugEventRepository cfDebugEventRepository) { super(partitionConfiguration, partitioningRepository, lcEventRepository, statsEventRepository, errorEventRepository, eventInsertRepository, ruleNodeDebugEventRepository, - ruleChainDebugEventRepository, logExecutor, statsFactory); + ruleChainDebugEventRepository, logExecutor, statsFactory, cfDebugEventRepository); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index 962be57892..9307b9e9be 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -25,6 +25,7 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventType; @@ -81,6 +82,9 @@ public class EventInsertRepository { insertStmtMap.put(EventType.DEBUG_RULE_CHAIN, "INSERT INTO " + EventType.DEBUG_RULE_CHAIN.getTable() + " (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.DEBUG_CALCULATED_FIELD, "INSERT INTO " + EventType.DEBUG_CALCULATED_FIELD.getTable() + + " (id, tenant_id, tsб entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); } public void save(List entities) { @@ -107,6 +111,8 @@ public class EventInsertRepository { return getRuleNodeEventSetter(events); case DEBUG_RULE_CHAIN: return getRuleChainEventSetter(events); + case DEBUG_CALCULATED_FIELD: + return getCalculatedFieldEventSetter(events); default: throw new RuntimeException(eventType + " support is not implemented!"); } @@ -206,6 +212,29 @@ public class EventInsertRepository { }; } + private BatchPreparedStatementSetter getCalculatedFieldEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + CalculatedFieldDebugEvent event = (CalculatedFieldDebugEvent) events.get(i); + setCommonEventFields(ps, event); + safePutUUID(ps, 6, event.getCalculatedFieldId().getId()); + safePutUUID(ps, 7, event.getEventEntity() != null ? event.getEventEntity().getId() : null); + safePutString(ps, 8, event.getEventEntity() != null ? event.getEventEntity().getEntityType().name() : null); + safePutUUID(ps, 9, event.getMsgId()); + safePutString(ps, 10, event.getMsgType()); + safePutString(ps, 11, event.getArguments()); + safePutString(ps, 12, event.getResult()); + safePutString(ps, 13, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + void safePutString(PreparedStatement ps, int parameterIdx, String value) throws SQLException { if (value != null) { ps.setString(parameterIdx, replaceNullChars(value)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index b8d5083402..e3c1b37536 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEventFilter; import org.thingsboard.server.common.data.event.ErrorEventFilter; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; @@ -72,6 +73,7 @@ public class JpaBaseEventDao implements EventDao { private final RuleChainDebugEventRepository ruleChainDebugEventRepository; private final ScheduledLogExecutorComponent logExecutor; private final StatsFactory statsFactory; + private final CalculatedFieldDebugEventRepository calculatedFieldDebugEventRepository; @Value("${sql.events.batch_size:10000}") private int batchSize; @@ -110,6 +112,7 @@ public class JpaBaseEventDao implements EventDao { repositories.put(EventType.ERROR, errorEventRepository); repositories.put(EventType.DEBUG_RULE_NODE, ruleNodeDebugEventRepository); repositories.put(EventType.DEBUG_RULE_CHAIN, ruleChainDebugEventRepository); + repositories.put(EventType.DEBUG_CALCULATED_FIELD, calculatedFieldDebugEventRepository); } @PreDestroy @@ -158,6 +161,8 @@ public class JpaBaseEventDao implements EventDao { return findEventByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, pageLink); case STATS: return findEventByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, pageLink); + case DEBUG_CALCULATED_FIELD: + return findEventByFilter(tenantId, entityId, (CalculatedFieldDebugEventFilter) eventFilter, pageLink); default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } @@ -193,6 +198,8 @@ public class JpaBaseEventDao implements EventDao { case STATS: removeEventsByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, startTime, endTime); break; + case DEBUG_CALCULATED_FIELD: + removeEventsByFilter(tenantId, entityId, (CalculatedFieldDebugEventFilter) eventFilter, startTime, endTime); default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } @@ -286,6 +293,27 @@ public class JpaBaseEventDao implements EventDao { ); } + private PageData findEventByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, TimePageLink pageLink) { + parseUUID(eventFilter.getCalculatedFieldId(), "Calculated Field Id"); + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + return DaoUtil.toPageData( + calculatedFieldDebugEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + UUID.fromString(eventFilter.getCalculatedFieldId()), + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.isError(), + eventFilter.getErrorStr(), + DaoUtil.toPageable(pageLink, EventEntity.eventColumnMap))); + } + private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, Long startTime, Long endTime) { ruleChainDebugEventRepository.removeEvents( tenantId, @@ -360,6 +388,25 @@ public class JpaBaseEventDao implements EventDao { ); } + private void removeEventsByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, Long startTime, Long endTime) { + parseUUID(eventFilter.getCalculatedFieldId(), "Calculated Field Id"); + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + calculatedFieldDebugEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + UUID.fromString(eventFilter.getCalculatedFieldId()), + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.isError(), + eventFilter.getErrorStr()); + } + @Override public List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) { return DaoUtil.convertDataList(getEventRepository(eventType).findLatestEvents(tenantId, entityId, limit)); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 51cc66c4ac..a6d1e8800d 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -934,6 +934,7 @@ CREATE TABLE IF NOT EXISTS calculated_field ( configuration_version int DEFAULT 0, configuration varchar(1000000), version BIGINT DEFAULT 1, + debug_settings varchar(1024), external_id UUID, CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) @@ -949,3 +950,19 @@ CREATE TABLE IF NOT EXISTS calculated_field_link ( configuration varchar(10000), CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE ); + +CREATE TABLE IF NOT EXISTS cf_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL , + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar, + cf_id uuid NOT NULL, + e_entity_id uuid, + e_entity_type varchar, + e_msg_id uuid, + e_msg_type varchar, + e_args varchar, + e_result varchar, + e_error varchar +) PARTITION BY RANGE (ts); From 069725a2b95dd22f8e7ab59250904c8d8897d11f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Jan 2025 10:43:26 +0200 Subject: [PATCH 118/416] WIP Refactoring of the cluster mode --- ...faultTbCalculatedFieldConsumerService.java | 245 ++++++++++++++++++ .../DefaultTbRuleEngineConsumerService.java | 5 + .../DefaultTelemetrySubscriptionService.java | 14 +- .../src/main/resources/thingsboard.yml | 16 +- .../server/common/util/ProtoUtils.java | 30 +-- common/proto/src/main/proto/queue.proto | 29 +++ .../queue/discovery/HashPartitionService.java | 15 +- .../server/queue/discovery/TopicService.java | 12 +- .../discovery/event/PartitionChangeEvent.java | 5 +- .../InMemoryMonolithQueueFactory.java | 34 +++ .../queue/provider/TbCoreQueueFactory.java | 11 +- .../provider/TbRuleEngineQueueFactory.java | 24 +- .../TbQueueCalculatedFieldSettings.java | 35 +++ 13 files changed, 430 insertions(+), 45 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java new file mode 100644 index 0000000000..85ee01ac5f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -0,0 +1,245 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; + +import java.util.List; +import java.util.UUID; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService implements TbCalculatedFieldConsumerService { + + @Value("${queue.calculated_fields.poll_interval}") + private long pollInterval; + @Value("${queue.calculated_fields.pack_processing_timeout}") + private long packProcessingTimeout; + @Value("${queue.calculated_fields.consumer_per_partition:true}") + private boolean consumerPerPartition; + @Value("${queue.calculated_fields.pool_size:8}") + private int poolSize; + + private final TbRuleEngineQueueFactory queueFactory; + + private final CalculatedFieldExecutionService calculatedFieldExecutionService; + + private MainQueueConsumerManager, CalculatedFieldQueueConfig> mainConsumer; + + private volatile ListeningExecutorService calculatedFieldsExecutor; + + public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, + ActorSystemContext actorContext, + TbDeviceProfileCache deviceProfileCache, + TbAssetProfileCache assetProfileCache, + TbTenantProfileCache tenantProfileCache, + TbApiUsageStateService apiUsageStateService, + PartitionService partitionService, + ApplicationEventPublisher eventPublisher, + JwtSettingsService jwtSettingsService, + CalculatedFieldExecutionService calculatedFieldExecutionService, + CalculatedFieldCache calculatedFieldCache) { + super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, + eventPublisher, jwtSettingsService); + this.queueFactory = tbQueueFactory; + this.calculatedFieldExecutionService = calculatedFieldExecutionService; + } + + @PostConstruct + public void init() { + super.init("tb-cf"); + this.calculatedFieldsExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(poolSize, "tb-cf-executor")); // TODO: multiple threads. + + this.mainConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() + .queueKey(new QueueKey(ServiceType.TB_CORE)) + .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .msgPackProcessor(this::processMsgs) + .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) + .consumerExecutor(consumersExecutor) + .scheduler(scheduler) + .taskExecutor(mgmtExecutor) + .build(); + } + + @PreDestroy + public void destroy() { + super.destroy(); + if (calculatedFieldsExecutor != null) { + calculatedFieldsExecutor.shutdownNow(); + } + } + + @Override + protected void startConsumers() { + super.startConsumers(); + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + log.debug("Subscribing to partitions: {}", event.getCalculatedFieldsPartitions()); + mainConsumer.update(event.getCalculatedFieldsPartitions()); + } + + private void processMsgs(List> msgs, TbQueueConsumer> consumer, CalculatedFieldQueueConfig config) throws Exception { + + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_RULE_ENGINE; + } + + @Override + protected long getNotificationPollDuration() { + return pollInterval; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected int getMgmtThreadPoolSize() { + return Math.max(Runtime.getRuntime().availableProcessors(), 4); + } + + @Override + protected TbQueueConsumer> createNotificationsConsumer() { + return queueFactory.createToCalculatedFieldNotificationsMsgConsumer(); + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { + ToCalculatedFieldNotificationMsg notification = msg.getValue(); + + callback.onSuccess(); + } + +// private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { +// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); +// var oldProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getOldProfileIdMSB(), profileUpdateMsg.getOldProfileIdLSB())); +// var newProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getNewProfileIdMSB(), profileUpdateMsg.getNewProfileIdLSB())); +// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); +// } +// +// private void processProfileEntityMsg(TransportProtos.ProfileEntityMsgProto profileEntityMsg) { +// var tenantId = toTenantId(profileEntityMsg.getTenantIdMSB(), profileEntityMsg.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityType(), new UUID(profileEntityMsg.getEntityIdMSB(), profileEntityMsg.getEntityIdLSB())); +// var profileId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityProfileType(), new UUID(profileEntityMsg.getProfileIdMSB(), profileEntityMsg.getProfileIdLSB())); +// boolean added = profileEntityMsg.getAdded(); +// Set entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); +// if (added) { +// entitiesByProfile.add(entityId); +// } else { +// entitiesByProfile.remove(entityId); +// } +// } +// +// private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldMsgProto calculatedFieldMsg, TbCallback callback) { +// var tenantId = toTenantId(calculatedFieldMsg.getTenantIdMSB(), calculatedFieldMsg.getTenantIdLSB()); +// var calculatedFieldId = new CalculatedFieldId(new UUID(calculatedFieldMsg.getCalculatedFieldIdMSB(), calculatedFieldMsg.getCalculatedFieldIdLSB())); +// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); +// DonAsynchron.withCallback(future, +// __ -> callback.onSuccess(), +// t -> { +// log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); +// callback.onFailure(t); +// }); +// } +// +// private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) { +// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); +// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); +// DonAsynchron.withCallback(future, +// __ -> callback.onSuccess(), +// t -> { +// log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); +// callback.onFailure(t); +// }); +// } +// +// private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { +// var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); +// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); +// DonAsynchron.withCallback(future, +// __ -> callback.onSuccess(), +// t -> { +// log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); +// callback.onFailure(t); +// }); +// } + + private void throwNotHandled(Object msg, TbCallback callback) { + log.warn("Message not handled: {}", msg); + callback.onFailure(new RuntimeException("Message not handled!")); + } + + private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { + return TenantId.fromUUID(new UUID(tenantIdMSB, tenantIdLSB)); + } + + @Override + protected void stopConsumers() { + super.stopConsumers(); + mainConsumer.stop(); + mainConsumer.awaitStop(); + } + + @Data(staticConstructor = "of") + public static class CalculatedFieldQueueConfig implements QueueConfig { + private final boolean consumerPerPartition; + private final int pollInterval; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 7d4d975cb4..5e07e33a9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -63,6 +63,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; + @Service @TbRuleEngineComponent @Slf4j @@ -107,6 +109,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { event.getPartitionsMap().forEach((queueKey, partitions) -> { + if (CALCULATED_FIELD_QUEUE_KEY.equals(queueKey)) { + return; + } if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { var consumer = getConsumer(queueKey).orElseGet(() -> { Queue config = queueService.findQueueByTenantIdAndName(queueKey.getTenantId(), queueKey.getQueueName()); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index dcb72b8dd0..fdf3ed6c50 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -149,13 +149,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } // We need to guarantee, that the message is successfully pushed to the calculated fields service before we execute any callbacks. - saveFuture = Futures.transformAsync(saveFuture, new AsyncFunction() { - @Override - public ListenableFuture apply(Integer input) throws Exception { - calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)); - return input; - } - }); +// saveFuture = Futures.transformAsync(saveFuture, new AsyncFunction() { +// @Override +// public ListenableFuture apply(Integer input) throws Exception { +// calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)); +// return input; +// } +// }); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 5151bc019b..94c1bef71c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1738,17 +1738,21 @@ queue: topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" - calculated-fields: - # Topic name for Calculated Field (CF) tasks - topic: "${TB_QUEUE_CF_TOPIC:tb_calculated_fields}" + calculated_fields: + # Topic name for Calculated Field (CF) events from Rule Engine + event_topic: "${TB_QUEUE_CF_EVENT_TOPIC:tb_cf_event}" + # Topic name for Calculated Field (CF) compacted states + state_topic: "${TB_QUEUE_CF_STATE_TOPIC:tb_cf_state}" # Interval in milliseconds to poll messages by CF (Rule Engine) microservices - poll-interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" + poll_interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" # Amount of partitions used by CF microservices partitions: "${TB_QUEUE_CF_PARTITIONS:10}" # Timeout for processing a message pack by CF microservices - pack-processing-timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" # Enable/disable a separate consumer per partition for CF queue - consumer-per-partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" + consumer_per_partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" + # Thread pool size for processing of the incoming messages + pool_size: "${TB_QUEUE_CF_POOL_SIZE:8}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 073f47d59b..1b743316bd 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -644,15 +644,15 @@ public class ProtoUtils { return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); } - public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { - if (telemetryProto.hasAttrKv()) { - return fromProto(telemetryProto.getAttrKv().getValue()); - } else if (telemetryProto.hasTsKv()) { - return fromProto(telemetryProto.getTsKv()); - } else { - throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); - } - } +// public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { +// if (telemetryProto.hasAttrKv()) { +// return fromProto(telemetryProto.getAttrKv().getValue()); +// } else if (telemetryProto.hasTsKv()) { +// return fromProto(telemetryProto.getTsKv()); +// } else { +// throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); +// } +// } public static TransportProtos.AttributeKey toAttributeKeyProto(String key, AttributeScope scope) { TransportProtos.AttributeKey.Builder builder = TransportProtos.AttributeKey.newBuilder(); @@ -673,12 +673,12 @@ public class ProtoUtils { return builder.build(); } - public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { - return TransportProtos.AttributeKvProto.newBuilder() - .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) - .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) - .build(); - } +// public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { +// return TransportProtos.AttributeKvProto.newBuilder() +// .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) +// .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) +// .build(); +// } public static TransportProtos.AttributeValueProto toAttributeValueProto(AttributeKvEntry attributeKvEntry) { TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder(); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4e719a02df..288c923aaa 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -819,6 +819,35 @@ message CalculatedFieldIdProto { int64 calculatedFieldIdLSB = 2; } +message SingleValueProto { + int64 ts = 1; + int64 version = 2; + KeyValueType type = 3; + bool has_v = 4; + bool bool_v = 5; + int64 long_v = 6; + double double_v = 7; + string string_v = 8; + string json_v = 9; +} + +message SingleValueArgumentProto { + string argName = 1; + SingleValueProto value = 2; +} + +message RollingArgumentProto { + string argName = 1; + repeated SingleValueProto values = 2; +} + +message CalculatedFieldStateProto { + CalculatedFieldEntityCtxIdProto id = 1; + // int32 version = 2; + repeated SingleValueArgumentProto singleValueArguments = 3; + repeated RollingArgumentProto rollingValueArguments = 4; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 53bdc78c93..7ac938f52f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -61,9 +61,11 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:10}") private Integer corePartitions; - @Value("${queue.calculated-fields.topic}") - private String cfTopic; - @Value("${queue.calculated-fields.partitions:10}") + @Value("${queue.calculated_fields.event_topic}") + private String cfEventTopic; + @Value("${queue.calculated_fields.state_topic}") + private String cfStateTopic; + @Value("${queue.calculated_fields.partitions:10}") private Integer cfPartitions; @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @@ -76,6 +78,8 @@ public class HashPartitionService implements PartitionService { @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; + public static final QueueKey CALCULATED_FIELD_QUEUE_KEY = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; private final TenantRoutingInfoService tenantRoutingInfoService; @@ -116,9 +120,8 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); - QueueKey cfKey = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); - partitionSizesMap.put(cfKey, cfPartitions); - partitionTopicsMap.put(cfKey, cfTopic); + partitionSizesMap.put(CALCULATED_FIELD_QUEUE_KEY, cfPartitions); + partitionTopicsMap.put(CALCULATED_FIELD_QUEUE_KEY, cfEventTopic); QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index 927c311a2d..8dec36c15c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -35,6 +35,7 @@ public class TopicService { private final ConcurrentMap tbCoreNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbRuleEngineNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbEdgeNotificationTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap tbCalculatedFieldNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentReferenceHashMap tbEdgeEventsNotificationTopics = new ConcurrentReferenceHashMap<>(); /** @@ -62,6 +63,11 @@ public class TopicService { return buildTopicPartitionInfo("tb_edge.notifications." + serviceId, null, null, false); } + public TopicPartitionInfo getCalculatedFieldNotificationsTopic(String serviceId) { + return tbCalculatedFieldNotificationTopics.computeIfAbsent(serviceId, + id -> buildNotificationsTopicPartitionInfo("calculated_field", serviceId)); + } + public TopicPartitionInfo getEdgeEventNotificationsTopic(TenantId tenantId, EdgeId edgeId) { return tbEdgeEventsNotificationTopics.computeIfAbsent(edgeId, id -> buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId)); } @@ -71,7 +77,11 @@ public class TopicService { } private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return buildTopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); + return buildNotificationsTopicPartitionInfo(serviceType.name().toLowerCase(), serviceId); + } + + private TopicPartitionInfo buildNotificationsTopicPartitionInfo(String serviceType, String serviceId) { + return buildTopicPartitionInfo(serviceType + ".notifications." + serviceId, null, null, false); } public TopicPartitionInfo buildTopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 3bb0c56f9a..57a4941981 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -23,10 +23,13 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.QueueKey; import java.io.Serial; +import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; + @ToString(callSuper = true) public class PartitionChangeEvent extends TbApplicationEvent { @@ -53,7 +56,7 @@ public class PartitionChangeEvent extends TbApplicationEvent { } public Set getCalculatedFieldsPartitions() { - return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); + return partitionsMap.getOrDefault(CALCULATED_FIELD_QUEUE_KEY, Collections.emptySet()); } private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index d70cad159b..c26e2d15c9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -33,6 +33,7 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -53,6 +54,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final InMemoryStorage storage; public InMemoryMonolithQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, @@ -62,6 +64,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueEdgeSettings edgeSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, InMemoryStorage storage) { this.topicService = topicService; this.coreSettings = coreSettings; @@ -71,6 +74,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.storage = storage; } @@ -139,6 +143,31 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + } + @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); @@ -209,6 +238,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index c4002f4d3e..0b3df5bccf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -159,4 +160,6 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } + TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index c406aeb311..76dad05393 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,9 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -109,11 +112,22 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory } /** - * Used to consume high priority messages by TB Core Service + * Used to consume high priority messages by TB Rule Engine Service * * @return */ TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); + + TbQueueConsumer> createToCalculatedFieldMsgConsumer(); + + TbQueueProducer> createToCalculatedFieldMsgProducer(); + + TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer(); + + TbQueueConsumer> createCalculatedFieldStateConsumer(); + + TbQueueProducer> createCalculatedFieldStateProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java new file mode 100644 index 0000000000..22bbd7e0f3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Data +@Component +public class TbQueueCalculatedFieldSettings { + + @Value("${queue.calculated_fields.event_topic}") + private String eventTopic; + + @Value("${queue.calculated_fields.state_topic}") + private String stateTopic; + + +} From e57746167dcdd582e4097e11fc8b6a9d06bbb0b0 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Thu, 23 Jan 2025 10:55:54 +0200 Subject: [PATCH 119/416] Update locale.constant-en_US.json --- ui-ngx/src/assets/locale/locale.constant-en_US.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index f19336d6ad..b5903a8fb6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -4907,7 +4907,7 @@ "alarm-severity-pattern-hint": "Use ${metadataKey} for value from metadata, $[messageKey] for value from message body. Alarm severity should be system (CRITICAL, MAJOR etc.)", "output-node-name-hint": "The rule node name corresponds to the relation type of the output message, and it is used to forward messages to other rule nodes in the caller rule chain.", "use-server-ts": "Use server timestamp", - "use-server-ts-hint": "Rule node will use the timestamp of message processing instead of the timestamp from the message. Useful for all sorts of sequential processing if you merge messages from multiple sources (devices, assets, etc).", + "use-server-ts-hint": "Use the server’s current timestamp for time series data that lacks an explicit timestamp. This helps maintain proper ordering when processing messages from multiple sources or when messages arrive out of sequence.", "kv-map-pattern-hint": "All input fields support templatization. Use $[messageKey] to extract value from the message and ${metadataKey} to extract value from the metadata.", "kv-map-single-pattern-hint": "Input field support templatization. Use $[messageKey] to extract value from the message and ${metadataKey} to extract value from the metadata.", "shared-scope": "Shared scope", @@ -5072,7 +5072,7 @@ "advanced-mode": "Advanced", "save-time-series": { "persistence-settings": "Persistence settings", - "persistence-settings-hint": "Persistence settings hint", + "persistence-settings-hint": "Define how and when time series data is saved. In Basic mode, apply a single persistence strategy to all actions or enable only WebSockets updates. Advanced mode allows you to configure individual persistence strategies for each action.", "strategy": "Strategy", "deduplication-interval": "Deduplication interval", "deduplication-interval-required": "Deduplication interval is required", From 4c71b9d5f68907dc229fa181a3c3f43aafcf84b7 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Jan 2025 13:56:44 +0200 Subject: [PATCH 120/416] WIP: Cluster mode implementation --- .../service/cf/CalculatedFieldCache.java | 4 +- .../cf/CalculatedFieldExecutionService.java | 12 +- .../cf/DefaultCalculatedFieldCache.java | 11 +- ...efaultCalculatedFieldExecutionService.java | 115 +++++++++++++++++- .../cf/ctx/state/CalculatedFieldCtx.java | 43 +++++++ ...faultTbCalculatedFieldConsumerService.java | 8 +- .../queue/DefaultTbClusterService.java | 12 +- .../TbCalculatedFieldConsumerService.java | 15 +++ .../AbstractSubscriptionService.java | 8 ++ .../DefaultTelemetrySubscriptionService.java | 38 +++--- .../server/cluster/TbClusterService.java | 3 + .../queue/discovery/HashPartitionService.java | 3 +- .../queue/discovery/PartitionService.java | 2 + .../queue/provider/TbCoreQueueFactory.java | 11 +- .../provider/TbCoreQueueProducerProvider.java | 16 +++ .../provider/TbQueueProducerProvider.java | 8 +- .../TbRuleEngineProducerProvider.java | 14 +++ .../provider/TbRuleEngineQueueFactory.java | 8 +- .../TbTransportQueueProducerProvider.java | 10 ++ .../TbVersionControlProducerProvider.java | 11 ++ 20 files changed, 298 insertions(+), 54 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 8730aeeedf..ea55894432 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -36,9 +36,9 @@ public interface CalculatedFieldCache { List getCalculatedFieldLinksByEntityId(EntityId entityId); - CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService); + List getCalculatedFieldCtxsByEntityId(EntityId entityId); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index e18c8b4119..2507546013 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; @@ -22,13 +24,13 @@ import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdat public interface CalculatedFieldExecutionService { /** - * Push incoming telemetry to the CF processing queue for async processing. - * @param request - telemetry request; - * @param callback - callback to be executed when the message is ack by the queue. + * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * @param request - telemetry save request; + * @param request - telemetry save result; */ - void pushRequestToQueue(CalculatedFieldTelemetryUpdateRequest request, TbCallback callback); + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); - void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); +// void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 868001d0d2..7e841a0cf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -36,6 +36,7 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -55,6 +56,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final CalculatedFieldService calculatedFieldService; private final AssetService assetService; private final DeviceService deviceService; + private final TbelInvokeService tbelInvokeService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -105,7 +107,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { calculatedFieldFetchLock.lock(); @@ -128,9 +130,12 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService) { + public List getCalculatedFieldCtxsByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } return getCalculatedFieldsByEntityId(entityId).stream() - .map(cf -> getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) + .map(cf -> getCalculatedFieldCtx(cf.getId())) .toList(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 84a21ab315..0f9b948a9d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -33,12 +33,13 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -60,6 +61,7 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; @@ -72,7 +74,9 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -103,6 +107,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -113,6 +119,16 @@ import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @RequiredArgsConstructor public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { + public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + } + }; + private final CalculatedFieldService calculatedFieldService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; @@ -121,7 +137,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final TimeseriesService timeseriesService; private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; - private final TbelInvokeService tbelInvokeService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -169,6 +184,70 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return "calculated-field-scheduled"; } + @Override + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; + //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; + //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); + if (send) { + clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); + } else { + if (callback != null) { + callback.onSuccess(null); + } + } + } + + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { + boolean send = false; + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); + if (!send) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); + } + if (!send) { + send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + .map(calculatedFieldCache::getCalculatedFieldCtx) + .anyMatch(linkedEntityFilter); + } + } + return send; + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ////TODO: IM to push to CF queue + return null; + } + + private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) + .forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + EntityId targetEntityId = ctx.getEntityId(); + + if (isProfileEntity(targetEntityId)) { + calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { + processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); + }); + } else { + processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); + } + }); + } + @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); @@ -374,7 +453,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { if (cfTargetEntityId != null) { - calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { + calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId).forEach(ctx -> { Map updatedTelemetry = request.getMappedTelemetry(ctx, cfTargetEntityId); if (!updatedTelemetry.isEmpty()) { EntityId targetEntityId = isProfileEntity(cfTargetEntityId) ? request.getEntityId() : cfTargetEntityId; @@ -391,7 +470,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); EntityId targetEntityId = ctx.getEntityId(); if (isProfileEntity(targetEntityId)) { @@ -830,4 +909,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } + private static TbQueueCallback wrap(FutureCallback callback) { + if (callback != null) { + return new FutureCallbackWrapper(callback); + } else { + return DUMMY_TB_QUEUE_CALLBACK; + } + } + + private static class FutureCallbackWrapper implements TbQueueCallback { + private final FutureCallback callback; + + public FutureCallbackWrapper(FutureCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index cb4052b7df..1a9f999497 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -20,15 +20,18 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -41,6 +44,9 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; + private final Map mainEntityArguments; + private final Map> linkedEntityArguments; + private final Map, String> referencedEntityKeys; private final List argNames; private Output output; @@ -55,6 +61,17 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); + this.mainEntityArguments = new HashMap<>(); + this.linkedEntityArguments = new HashMap<>(); + for (Map.Entry entry : arguments.entrySet()) { + var refId = entry.getValue().getRefEntityId(); + var refKey = entry.getValue().getRefEntityKey(); + if (refId == null) { + mainEntityArguments.put(refKey, entry.getKey()); + } else { + linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); + } + } this.referencedEntityKeys = arguments.entrySet().stream() .collect(Collectors.toMap( entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), @@ -82,4 +99,30 @@ public class CalculatedFieldCtx { ); } + public boolean matches(List values) { + return matches(mainEntityArguments, values); + } + + public boolean linkMatches(EntityId entityId, List values) { + var map = linkedEntityArguments.get(entityId); + if (map == null) { + return false; + } else { + return matches(map, values); + } + } + + private static boolean matches(Map argMap, List values) { + for (TsKvEntry tsKv : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); + if (argMap.containsKey(latestKey)) { + return true; + } + ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); + if (argMap.containsKey(rollingKey)) { + return true; + } + } + return false; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 85ee01ac5f..ff4634f957 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 551b0ddbc2..725e0fdd79 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -69,7 +69,8 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -108,7 +109,9 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; import static org.thingsboard.server.common.util.ProtoUtils.toProto; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -336,6 +339,13 @@ public class DefaultTbClusterService implements TbClusterService { toTransportNfs.incrementAndGet(); } + @Override + public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + @Override public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java index 387bdd7143..bba4b1e35b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index 0bb8f76398..21cb902ad6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -116,4 +116,12 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList }, executor); } + protected static Consumer safeCallback(FutureCallback callback) { + if (callback != null) { + return callback::onFailure; + } else { + return throwable -> {}; + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index c9602b4650..71716a16e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.telemetry; -import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -53,7 +52,6 @@ import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -149,14 +147,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } else { resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } - addMainCallback(resultFuture, request.getCallback()); + DonAsynchron.withCallback(resultFuture, result -> { + calculatedFieldExecutionService.pushRequestToQueue(request, result); + }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. - //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest - addCallback(resultFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); return resultFuture; } @@ -171,6 +168,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); + //TODO: IM to push to CF queue addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } @@ -268,27 +266,21 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes)); } private void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toAttributesDeleteProto(tenantId, entityId, scope, keys, notifyDevice); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toAttributesDeleteProto(tenantId, entityId, scope, keys, notifyDevice)); } private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts)); } private void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List keys, List ts) { @@ -308,9 +300,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, updated, TbCallback.EMPTY); subscriptionManagerService.onTimeSeriesDelete(tenantId, entityId, deleted, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toTimeseriesDeleteProto(tenantId, entityId, keys); - }); + }, () -> TbSubscriptionUtils.toTimeseriesDeleteProto(tenantId, entityId, keys)); } private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 69334b774f..151d093c4f 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -76,6 +77,8 @@ public interface TbClusterService extends TbQueueClusterService { void pushNotificationToTransport(String targetServiceId, ToTransportMsg response, TbQueueCallback callback); + void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 7ac938f52f..5c6bf545b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -323,7 +323,8 @@ public class HashPartitionService implements PartitionService { } } - private TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { + @Override + public TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { Integer partitionSize = partitionSizesMap.get(queueKey); if (partitionSize == null) { throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b5744981bd..8e2152830e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -37,6 +37,8 @@ public interface PartitionService { TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId); + List resolveAll(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); boolean isMyPartition(ServiceType serviceType, TenantId tenantId, EntityId entityId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 0b3df5bccf..69b3dff1f0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -160,6 +161,8 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } + TbQueueProducer> createToCalculatedFieldMsgProducer(); + TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 9cf18e6cb4..77f3cbcdfe 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -17,6 +17,8 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -48,6 +50,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; + private TbQueueProducer> toCalculatedFields; + private TbQueueProducer> toCalculatedFieldNotifications; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -66,6 +70,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); + this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); + this.toCalculatedFieldNotifications = tbQueueProvider.createToCalculatedFieldNotificationMsgProducer(); } @Override @@ -124,4 +130,14 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { return toEdgeEvents; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + return toCalculatedFields; + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + return toCalculatedFieldNotifications; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index ec31763baa..2a151d58ed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -76,7 +78,7 @@ public interface TbQueueProducerProvider { */ TbQueueProducer> getTbUsageStatsMsgProducer(); - /** + /** * Used to push messages to other instances of TB Core Service * * @return @@ -91,4 +93,8 @@ public interface TbQueueProducerProvider { TbQueueProducer> getTbEdgeEventsMsgProducer(); + TbQueueProducer> getCalculatedFieldsMsgProducer(); + + TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index e9f7773a26..6a73dce1cb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -18,6 +18,8 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -47,6 +49,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toEdge; private TbQueueProducer> toEdgeNotifications; private TbQueueProducer> toEdgeEvents; + private TbQueueProducer> toCalculatedFields; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -64,6 +67,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); + this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); } @Override @@ -121,4 +125,14 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + return toCalculatedFields; + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine Service!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 76dad05393..d3c1d09399 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 7960cbcf32..88b0f0b1ef 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -111,4 +112,13 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index cd4fa12df0..09c835e5fe 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -107,4 +108,14 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + } From 80593f4b52d09703e397bb20799d9501baed1f60 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Jan 2025 14:05:41 +0200 Subject: [PATCH 121/416] UI: Fixed validation and style in Save ts rule node config --- ...ced-persistence-setting-row.component.html | 1 + .../action/timeseries-config.component.html | 15 ++++++------- .../action/timeseries-config.component.ts | 8 +++---- .../action/timeseries-config.models.ts | 2 +- .../common/time-unit-input.component.html | 2 +- .../common/time-unit-input.component.ts | 21 ++++++++++++++----- 6 files changed, 31 insertions(+), 18 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html index 038fe41bb2..2bb51ad504 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html @@ -32,6 +32,7 @@ requiredText="{{ 'rule-node-config.save-time-series.deduplication-interval-required' | translate }}" minErrorText="{{ 'rule-node-config.save-time-series.deduplication-interval-min-max-range' | translate }}" maxErrorText="{{ 'rule-node-config.save-time-series.deduplication-interval-min-max-range' | translate }}" + [minTime]="1" [maxTime]="maxDeduplicateTime" formControlName="deduplicationIntervalSecs"> diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html index 47655c1285..eb2811c8f8 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html @@ -45,6 +45,7 @@ minErrorText="{{ 'rule-node-config.save-time-series.deduplication-interval-min-max-range' | translate }}" maxErrorText="{{ 'rule-node-config.save-time-series.deduplication-interval-min-max-range' | translate }}" [maxTime]="maxDeduplicateTime" + [minTime]="1" formControlName="deduplicationIntervalSecs"> } @@ -56,12 +57,18 @@ > } -

+
rule-node-config.advanced-settings +
+ + {{ 'rule-node-config.use-server-ts' | translate }} + +
-
- - {{ 'rule-node-config.use-server-ts' | translate }} - -
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts index a834d0d43f..4a061d3193 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts @@ -60,16 +60,16 @@ export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { if (config?.persistenceSettings) { const isAdvanced = config?.persistenceSettings?.type === PersistenceType.ADVANCED; persistenceSettings = { - ...config.persistenceSettings, - isAdvanced: isAdvanced, type: isAdvanced ? PersistenceType.ON_EVERY_MESSAGE : config.persistenceSettings.type, + isAdvanced: isAdvanced, + deduplicationIntervalSecs: config.persistenceSettings?.deduplicationIntervalSecs ?? 60, advanced: isAdvanced ? config.persistenceSettings : defaultAdvancedPersistenceStrategy } } else { persistenceSettings = { type: PersistenceType.ON_EVERY_MESSAGE, isAdvanced: false, - deduplicationIntervalSecs: 10, + deduplicationIntervalSecs: 60, advanced: defaultAdvancedPersistenceStrategy }; } @@ -104,7 +104,7 @@ export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { isAdvanced: [config?.persistenceSettings?.isAdvanced ?? false], type: [config?.persistenceSettings?.type ?? PersistenceType.ON_EVERY_MESSAGE], deduplicationIntervalSecs: [ - {value: config?.persistenceSettings?.deduplicationIntervalSecs ?? 10, disabled: true}, + {value: config?.persistenceSettings?.deduplicationIntervalSecs ?? 60, disabled: true}, [Validators.required, Validators.max(maxDeduplicateTimeSecs)] ], advanced: [{value: null, disabled: true}] diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts index 241787d511..f70e8548b1 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts @@ -16,7 +16,7 @@ import { DAY, SECOND } from '@shared/models/time/time.models'; -export const maxDeduplicateTimeSecs = DAY / SECOND + 1; +export const maxDeduplicateTimeSecs = DAY / SECOND; export interface TimeseriesNodeConfiguration { persistenceSettings: PersistenceSettings; diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html index 1fd0a60159..9f65a2d9fc 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html @@ -33,7 +33,7 @@ {{ maxErrorText }} - + rule-node-config.units @for (timeUnit of timeUnits; track timeUnit) { diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts index 3dc0c00124..08d501477c 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts @@ -22,7 +22,8 @@ import { NG_VALIDATORS, NG_VALUE_ACCESSOR, ValidationErrors, - Validator, Validators + Validator, + Validators } from '@angular/forms'; import { TimeUnit, timeUnitTranslations } from '../rule-node-config.models'; import { isDefinedAndNotNull, isNumeric } from '@core/utils'; @@ -60,6 +61,10 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, @Input() requiredText: string; + @Input() + @coerceNumber() + minTime = 0; + @Input() minErrorText: string; @@ -75,7 +80,7 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, timeUnitTranslations = timeUnitTranslations; timeInputForm = this.fb.group({ - time: [0, Validators.min(0)], + time: [0], timeUnit: [TimeUnit.SECONDS] }); @@ -97,7 +102,7 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, ngOnInit() { if(this.required || this.maxTime) { const timeControl = this.timeInputForm.get('time'); - const validators = []; + const validators = [Validators.pattern(/^\d*$/)]; if (this.required) { validators.push(Validators.required); } @@ -106,6 +111,9 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, Validators.max(Math.floor(this.maxTime / this.timeIntervalsInSec.get(this.timeInputForm.get('timeUnit').value)))(control) ); } + if (isDefinedAndNotNull(this.minTime)) { + validators.push(Validators.min(this.minTime)); + } timeControl.setValidators(validators); timeControl.updateValueAndValidity({ emitEvent: false }); @@ -137,6 +145,9 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, this.timeInputForm.disable({emitEvent: false}); } else { this.timeInputForm.enable({emitEvent: false}); + if(this.timeInputForm.invalid) { + setTimeout(() => this.updatedModel(this.timeInputForm.value, true)) + } } } @@ -161,9 +172,9 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, }; } - private updatedModel(value: Partial) { + private updatedModel(value: Partial, forceUpdated = false) { const time = value.time * this.timeIntervalsInSec.get(value.timeUnit); - if (this.modelValue !== time) { + if (this.modelValue !== time || forceUpdated) { this.modelValue = time; this.propagateChange(time); } From 2cc0d6f513bdd4f5873e62af1db51d5d6b10eab4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 23 Jan 2025 15:49:31 +0200 Subject: [PATCH 122/416] added implementations for consumer/producer methods --- ...faultTbCalculatedFieldConsumerService.java | 8 +- .../TbCalculatedFieldConsumerService.java | 15 ++++ .../src/main/resources/thingsboard.yml | 2 + .../queue/kafka/TbKafkaTopicConfigs.java | 5 ++ .../provider/KafkaMonolithQueueFactory.java | 81 +++++++++++++++++++ .../provider/KafkaTbCoreQueueFactory.java | 15 ++++ .../KafkaTbRuleEngineQueueFactory.java | 72 ++++++++++++++++- .../queue/provider/TbCoreQueueFactory.java | 8 +- .../provider/TbRuleEngineQueueFactory.java | 8 +- .../dao/sql/event/EventInsertRepository.java | 2 +- 10 files changed, 202 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 85ee01ac5f..ff4634f957 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java index 387bdd7143..bba4b1e35b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 94c1bef71c..fdd9923292 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1619,6 +1619,8 @@ queue: edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge event topic edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Calculated Field topics + calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index ee529e8a68..cdd0add38b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -52,6 +52,8 @@ public class TbKafkaTopicConfigs { private String housekeeperProperties; @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") private String housekeeperReprocessingProperties; + @Value("${queue.kafka.topic-properties.calculated-field:}") + private String calculatedFieldProperties; @Getter private Map coreConfigs; @@ -79,6 +81,8 @@ public class TbKafkaTopicConfigs { private Map edgeConfigs; @Getter private Map edgeEventConfigs; + @Getter + private Map calculatedFieldConfigs; @PostConstruct private void init() { @@ -97,6 +101,7 @@ public class TbKafkaTopicConfigs { housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); + calculatedFieldConfigs = PropertyUtils.getProps(calculatedFieldProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index dd5d61e834..01e174023c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,6 +25,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -54,6 +57,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -79,6 +83,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueVersionControlSettings vcSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; @@ -94,6 +99,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -106,6 +112,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbQueueVersionControlSettings vcSettings, TbQueueEdgeSettings edgeSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; @@ -119,6 +126,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -133,6 +141,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); } @Override @@ -490,6 +499,75 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + consumerBuilder.clientId("monolith-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId())); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + consumerBuilder.clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -522,5 +600,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi if (edgeAdmin != null) { edgeAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index cc0e044917..aceccf7e58 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -53,6 +54,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -79,6 +81,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -107,6 +110,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -119,6 +123,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -439,6 +444,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-calculated-field-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 87a1a69c2e..46b35f9acd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,6 +23,9 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -49,6 +52,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -71,6 +75,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -81,6 +86,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -90,7 +96,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, + TbQueueEdgeSettings edgeSettings, TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -101,6 +107,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -111,6 +118,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); } @Override @@ -293,6 +301,65 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { .build(); } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + consumerBuilder.clientId("tb-rule-engine-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("tb-calculated-field-notifications-node-") + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + consumerBuilder.clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -313,5 +380,8 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 0b3df5bccf..673ac3a434 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 76dad05393..d3c1d09399 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index 9307b9e9be..bdb675cbb7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -83,7 +83,7 @@ public class EventInsertRepository { " (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); insertStmtMap.put(EventType.DEBUG_CALCULATED_FIELD, "INSERT INTO " + EventType.DEBUG_CALCULATED_FIELD.getTable() + - " (id, tenant_id, tsб entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + + " (id, tenant_id, ts, entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); } From e8dbe562191da733e29530dc1f452a41f4535776 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Jan 2025 17:01:21 +0200 Subject: [PATCH 123/416] UI: Fixed advanced settings style in Save ts rule node config --- .../rule-node/action/timeseries-config.component.html | 3 ++- .../rule-node/common/time-unit-input.component.html | 4 ++-- .../components/rule-node/common/time-unit-input.component.ts | 4 ++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html index eb2811c8f8..e803d7443d 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html @@ -57,7 +57,7 @@ > } -

+
rule-node-config.advanced-settings @@ -71,6 +71,7 @@ - + {{ requiredText }} @@ -33,7 +33,7 @@ {{ maxErrorText }} - + rule-node-config.units @for (timeUnit of timeUnits; track timeUnit) { diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts index 08d501477c..11358ae6eb 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts @@ -30,6 +30,7 @@ import { isDefinedAndNotNull, isNumeric } from '@core/utils'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { coerceBoolean, coerceNumber } from '@shared/decorators/coercion'; import { DAY, HOUR, MINUTE, SECOND } from '@shared/models/time/time.models'; +import { SubscriptSizing } from '@angular/material/form-field'; interface TimeUnitInputModel { time: number; @@ -75,6 +76,9 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, @Input() maxErrorText: string; + @Input() + subscriptSizing: SubscriptSizing = 'fixed'; + timeUnits = Object.values(TimeUnit).filter(item => item !== TimeUnit.MILLISECONDS) as TimeUnit[]; timeUnitTranslations = timeUnitTranslations; From 564162644360fddc6e37c7e29d485d72b88dc1e5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 24 Jan 2025 10:53:13 +0200 Subject: [PATCH 124/416] added processNotification impl --- .../cf/CalculatedFieldExecutionService.java | 14 +- ...efaultCalculatedFieldExecutionService.java | 309 ++++++++---------- .../cf/ctx/state/CalculatedFieldCtx.java | 29 +- ...faultTbCalculatedFieldConsumerService.java | 71 ++-- .../queue/DefaultTbClusterService.java | 36 +- .../queue/DefaultTbCoreConsumerService.java | 37 --- .../DefaultTelemetrySubscriptionService.java | 4 +- .../server/cluster/TbClusterService.java | 2 + .../server/common/util/ProtoUtils.java | 94 ------ .../provider/KafkaTbCoreQueueFactory.java | 16 + .../queue/provider/TbCoreQueueFactory.java | 4 +- 11 files changed, 245 insertions(+), 371 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 2507546013..1ad0377e27 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; public interface CalculatedFieldExecutionService { @@ -30,18 +32,18 @@ public interface CalculatedFieldExecutionService { */ void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); + void pushRequestToQueue(AttributesSaveRequest request); + // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ - void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); - void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); - - void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); +// void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); - void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); + void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0f8cdd4fd4..cbd5445cdf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -33,17 +33,15 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -74,7 +72,14 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; @@ -87,15 +92,12 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -112,7 +114,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @Slf4j @@ -195,7 +196,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); } - private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { + @Override + public void pushRequestToQueue(AttributesSaveRequest request) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request), request.getCallback()); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); if (send) { clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); @@ -223,31 +232,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return send; } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { - ////TODO: IM to push to CF queue - return null; - } - - private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - - calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) - .forEach(link -> { - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - EntityId targetEntityId = ctx.getEntityId(); - - if (isProfileEntity(targetEntityId)) { - calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); - }); - } else { - processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); - } - }); - } - @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); @@ -318,19 +302,20 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { + public void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - if (proto.getDeleted()) { + ComponentLifecycleEvent event = proto.getEvent(); + if (ComponentLifecycleEvent.DELETED.equals(event)) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); calculatedFieldCache.evict(calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (proto.getUpdated()) { + if (ComponentLifecycleEvent.UPDATED.equals(event)) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); if (!shouldReinit) { @@ -340,14 +325,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (cf != null) { calculatedFieldCache.addCalculatedField(tenantId, calculatedFieldId); EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); initializeStateForEntity(calculatedFieldCtx, entityId, callback); } case ASSET_PROFILE, DEVICE_PROFILE -> { - log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); + log.info("Initializing state for all entities in profile: tenantICalculatedFieldMsgProtod=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> entry.getValue().getRefEntityId() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -492,30 +477,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - @Override - public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); - - if (proto.getLinksList().isEmpty()) { - onTelemetryUpdate(request); - return; - } - - proto.getLinksList().forEach(ctxIdProto -> { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } catch (Exception e) { - log.trace("Failed to process telemetry update msg: [{}]", proto, e); - } - } +// @Override +// public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { +// try { +// CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); +// +// if (proto.getLinksList().isEmpty()) { +// onTelemetryUpdate(request); +// return; +// } +// +// proto.getLinksList().forEach(ctxIdProto -> { +// CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); +// CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); +// +// Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); +// if (!updatedTelemetry.isEmpty()) { +// EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); +// executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); +// } +// }); +// } catch (Exception e) { +// log.trace("Failed to process telemetry update msg: [{}]", proto, e); +// } +// } private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); @@ -526,50 +511,41 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); - EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); - initializeStateForEntityByProfile(entityId, newProfileId, callback); - } else { - clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setEntityProfileUpdateMsg(proto).build(), null); - } - } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); - } - } - - @Override - public void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { - log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); + EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); - calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).forEach(cf -> clearState(cf.getId(), entityId)); - } else { - log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntityByProfile(entityId, profileId, callback); + calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); + } + if (proto.getAdded()) { + log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); + + EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); + initializeStateForEntityByProfile(entityId, newProfileId, callback); + } + if (proto.getUpdated()) { + log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); + + EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); + EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); + + calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); + initializeStateForEntityByProfile(entityId, newProfileId, callback); } } else { - clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); + clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); } } catch (Exception e) { - log.trace("Failed to process profile entity msg: [{}]", proto, e); + log.trace("Failed to process entity update msg: [{}]", proto, e); } } @@ -582,7 +558,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).stream() - .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) + .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId())) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -775,88 +751,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { - return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); - } - - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( - CalculatedFieldTelemetryUpdateRequest request, List links - ) { - TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); - - builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()) - .setEntityType(request.getEntityId().getEntityType().name()) - .setEntityIdMSB(request.getEntityId().getId().getMostSignificantBits()) - .setEntityIdLSB(request.getEntityId().getId().getLeastSignificantBits()); - - for (CalculatedFieldEntityCtxId link : links) { - builder.addLinks(toProto(link)); - } - - for (CalculatedFieldId calculatedFieldId : request.getPreviousCalculatedFieldIds()) { - builder.addPreviousCalculatedFields(toProto(calculatedFieldId)); - } - - if (request instanceof CalculatedFieldAttributeUpdateRequest attributeUpdateRequest) { - builder.setScope(attributeUpdateRequest.getScope().name()); - } - - for (KvEntry entry : request.getKvEntries()) { - TransportProtos.TelemetryProto.Builder telemetryBuilder = TransportProtos.TelemetryProto.newBuilder(); - if (request instanceof CalculatedFieldTimeSeriesUpdateRequest) { - telemetryBuilder.setTsKv(toTsKvProto((TsKvEntry) entry)); - } - if (request instanceof CalculatedFieldAttributeUpdateRequest attrRequest) { - telemetryBuilder.setAttrKv(ProtoUtils.toAttributeKvProto((AttributeKvEntry) entry, attrRequest.getScope())); - } - builder.addUpdatedTelemetry(telemetryBuilder.build()); - } - - return builder.build(); - } - - private CalculatedFieldTelemetryUpdateRequest fromProto(TransportProtos.TelemetryUpdateMsgProto proto) { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - - List updatedTelemetry = proto.getUpdatedTelemetryList().stream() - .map(ProtoUtils::fromTelemetryProto) - .toList(); - - boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); - - return attributesUpdated - ? new CalculatedFieldAttributeUpdateRequest( - tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()) - : new CalculatedFieldTimeSeriesUpdateRequest( - tenantId, entityId, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); - } - - private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - - private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { - return TransportProtos.CalculatedFieldIdProto.newBuilder() - .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) - .build(); - } +// private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { +// return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() +// .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) +// .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) +// .setEntityType(ctxId.entityId().getEntityType().name()) +// .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) +// .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) +// .build(); +// } +// +// private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { +// return TransportProtos.CalculatedFieldIdProto.newBuilder() +// .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) +// .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) +// .build(); +// } private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); @@ -905,6 +815,51 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + for (TsKvEntry entry : request.getEntries()) { + telemetryMsg.addTsData(ProtoUtils.toTsKvProto(entry)); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); + for (AttributeKvEntry entry : request.getEntries()) { + telemetryMsg.addAttrData(ProtoUtils.toProto(entry)); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds) { + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); + + telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + + telemetryMsg.setEntityType(entityId.getEntityType().name()); + telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + + for (CalculatedFieldId cfId : calculatedFieldIds) { + CalculatedFieldIdProto.Builder calculatedFieldIdProto = CalculatedFieldIdProto.newBuilder(); + calculatedFieldIdProto.setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()); + calculatedFieldIdProto.setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()); + telemetryMsg.addPreviousCalculatedFields(calculatedFieldIdProto.build()); + } + + return telemetryMsg; + } + private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 1a9f999497..e4abbee4cd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; @@ -99,20 +101,35 @@ public class CalculatedFieldCtx { ); } + public boolean matches(List values, AttributeScope scope) { + return matchesAttributes(mainEntityArguments, values, scope); + } + + public boolean linkMatches(EntityId entityId, List values, AttributeScope scope) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesAttributes(map, values, scope); + } + public boolean matches(List values) { - return matches(mainEntityArguments, values); + return matchesTimeSeries(mainEntityArguments, values); } public boolean linkMatches(EntityId entityId, List values) { var map = linkedEntityArguments.get(entityId); - if (map == null) { - return false; - } else { - return matches(map, values); + return map != null && matchesTimeSeries(map, values); + } + + private static boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + for (AttributeKvEntry attrKv : values) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); + if (argMap.containsKey(attrKey)) { + return true; + } } + return false; } - private static boolean matches(Map argMap, List values) { + private boolean matchesTimeSeries(Map argMap, List values) { for (TsKvEntry tsKv : values) { ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); if (argMap.containsKey(latestKey)) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index ff4634f957..4eb3f0cd89 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.queue; +import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; @@ -24,13 +25,17 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -157,8 +162,12 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { - ToCalculatedFieldNotificationMsg notification = msg.getValue(); - + ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); + if (toCfNotification.hasComponentLifecycle()) { + forwardToCalculatedFieldService(toCfNotification.getComponentLifecycle(), callback); + } else if (toCfNotification.hasEntityUpdateMsg()) { + forwardToCalculatedFieldService(toCfNotification.getEntityUpdateMsg(), callback); + } callback.onSuccess(); } @@ -184,41 +193,29 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer // } // } // -// private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldMsgProto calculatedFieldMsg, TbCallback callback) { -// var tenantId = toTenantId(calculatedFieldMsg.getTenantIdMSB(), calculatedFieldMsg.getTenantIdLSB()); -// var calculatedFieldId = new CalculatedFieldId(new UUID(calculatedFieldMsg.getCalculatedFieldIdMSB(), calculatedFieldMsg.getCalculatedFieldIdLSB())); -// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); -// DonAsynchron.withCallback(future, -// __ -> callback.onSuccess(), -// t -> { -// log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); -// callback.onFailure(t); -// }); -// } -// -// private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) { -// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); -// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); -// DonAsynchron.withCallback(future, -// __ -> callback.onSuccess(), -// t -> { -// log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); -// callback.onFailure(t); -// }); -// } -// -// private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { -// var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); -// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); -// DonAsynchron.withCallback(future, -// __ -> callback.onSuccess(), -// t -> { -// log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); -// callback.onFailure(t); -// }); -// } + private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + + private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process entity updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } private void throwNotHandled(Object msg, TbCallback callback) { log.warn("Message not handled: {}", msg); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 725e0fdd79..b510b813d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -69,8 +69,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -80,6 +79,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.QueueDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.QueueUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos.ResourceDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.ResourceUpdateMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -109,7 +110,6 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; import static org.thingsboard.server.common.util.ProtoUtils.toProto; import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @@ -346,6 +346,13 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + @Override public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); @@ -809,16 +816,23 @@ public class DefaultTbClusterService implements TbClusterService { } private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { - TransportProtos.CalculatedFieldMsgProto.Builder builder = TransportProtos.CalculatedFieldMsgProto.newBuilder(); + ComponentLifecycleMsgProto.Builder builder = ComponentLifecycleMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()); - builder.setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); - builder.setAdded(added); - builder.setUpdated(updated); - builder.setDeleted(deleted); - TransportProtos.CalculatedFieldMsgProto msg = builder.build(); - pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); + builder.setEntityType(TransportProtos.EntityTypeProto.CALCULATED_FIELD); + builder.setEntityIdMSB(calculatedFieldId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); + TransportProtos.ComponentLifecycleEvent event; + if (added) { + event = TransportProtos.ComponentLifecycleEvent.CREATED; + } else if (updated) { + event = TransportProtos.ComponentLifecycleEvent.UPDATED; + } else { + event = TransportProtos.ComponentLifecycleEvent.DELETED; + } + builder.setEvent(event); + + pushNotificationToCalculatedFields(tenantId, calculatedFieldId, ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(builder).build(), null); } private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index c598540ff2..e3a1ca22d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -38,7 +38,6 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -702,42 +701,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) { - var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { - var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 71716a16e9..0f13dbb2f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -168,7 +168,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); - //TODO: IM to push to CF queue + DonAsynchron.withCallback(saveFuture, result -> { + calculatedFieldExecutionService.pushRequestToQueue(request); + }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 151d093c4f..b28bf73c89 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -79,6 +79,8 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 1b743316bd..3bace4d91f 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -22,7 +22,6 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -59,7 +58,6 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; @@ -630,98 +628,6 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } - public static KvEntry fromProto(TransportProtos.TsKvProto proto) { - TransportProtos.KeyValueProto kvProto = proto.getKv(); - String key = kvProto.getKey(); - KvEntry entry = switch (kvProto.getType()) { - case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); - case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); - case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); - case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); - case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); - default -> null; - }; - return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); - } - -// public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { -// if (telemetryProto.hasAttrKv()) { -// return fromProto(telemetryProto.getAttrKv().getValue()); -// } else if (telemetryProto.hasTsKv()) { -// return fromProto(telemetryProto.getTsKv()); -// } else { -// throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); -// } -// } - - public static TransportProtos.AttributeKey toAttributeKeyProto(String key, AttributeScope scope) { - TransportProtos.AttributeKey.Builder builder = TransportProtos.AttributeKey.newBuilder(); - builder.setAttributeKey(key); - switch (scope) { - case CLIENT_SCOPE: - builder.setScope(TransportProtos.AttributeScopeProto.CLIENT_SCOPE); - break; - case SERVER_SCOPE: - builder.setScope(TransportProtos.AttributeScopeProto.SERVER_SCOPE); - break; - case SHARED_SCOPE: - builder.setScope(TransportProtos.AttributeScopeProto.SHARED_SCOPE); - break; - default: - throw new IllegalArgumentException("Unsupported attribute scope: " + scope); - } - return builder.build(); - } - -// public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { -// return TransportProtos.AttributeKvProto.newBuilder() -// .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) -// .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) -// .build(); -// } - - public static TransportProtos.AttributeValueProto toAttributeValueProto(AttributeKvEntry attributeKvEntry) { - TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder(); - builder.setLastUpdateTs(attributeKvEntry.getLastUpdateTs()); - switch (attributeKvEntry.getDataType()) { - case BOOLEAN: - builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) - .setHasV(true) - .setBoolV(attributeKvEntry.getBooleanValue().orElse(false)); - break; - case LONG: - builder.setType(TransportProtos.KeyValueType.LONG_V) - .setHasV(true) - .setLongV(attributeKvEntry.getLongValue().orElse(0L)); - break; - case DOUBLE: - builder.setType(TransportProtos.KeyValueType.DOUBLE_V) - .setHasV(true) - .setDoubleV(attributeKvEntry.getDoubleValue().orElse(0.0)); - break; - case STRING: - builder.setType(TransportProtos.KeyValueType.STRING_V) - .setHasV(true) - .setStringV(attributeKvEntry.getStrValue().orElse("")); - break; - case JSON: - builder.setType(TransportProtos.KeyValueType.JSON_V) - .setHasV(true) - .setJsonV(attributeKvEntry.getJsonValue().orElse("{}")); - break; - default: - builder.setHasV(false); - throw new IllegalArgumentException("Unsupported AttributeKvEntry data type: " + attributeKvEntry.getDataType()); - } - if (attributeKvEntry.getKey() != null) { - builder.setKey(attributeKvEntry.getKey()); - } - if (attributeKvEntry.getVersion() != null) { - builder.setVersion(attributeKvEntry.getVersion()); - } - return builder.build(); - } - public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { return TransportProtos.TsKvProto.newBuilder() .setTs(tsKvEntry.getTs()) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index aceccf7e58..329f1d7544 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -96,6 +97,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -138,6 +140,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); } @Override @@ -444,6 +447,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + @Override public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); @@ -483,5 +496,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { if (vcAdmin != null) { vcAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 69b3dff1f0..1ece8249f2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -18,7 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -161,7 +161,7 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } - TbQueueProducer> createToCalculatedFieldMsgProducer(); + TbQueueProducer> createToCalculatedFieldMsgProducer(); TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); From 85119d02475a5cbfb6688bc609414f2d3d5bf5e0 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Jan 2025 11:37:32 +0200 Subject: [PATCH 125/416] WIP: cluster mode implementation --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 11 +++-- ...faultTbCalculatedFieldConsumerService.java | 6 +-- .../queue/DefaultTbClusterService.java | 42 +++++++------------ .../DefaultTelemetrySubscriptionService.java | 3 +- .../src/main/resources/thingsboard.yml | 2 +- 6 files changed, 29 insertions(+), 39 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 1ad0377e27..93a0ecc75f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -23,6 +23,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntit import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; +import java.util.List; + public interface CalculatedFieldExecutionService { /** @@ -32,7 +34,7 @@ public interface CalculatedFieldExecutionService { */ void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); - void pushRequestToQueue(AttributesSaveRequest request); + void pushRequestToQueue(AttributesSaveRequest request, List result); // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index cbd5445cdf..d46a7515ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -197,14 +197,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void pushRequestToQueue(AttributesSaveRequest request) { + public void pushRequestToQueue(AttributesSaveRequest request, List result) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), - () -> toCalculatedFieldTelemetryMsgProto(request), request.getCallback()); + () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); } - private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, + Predicate mainEntityFilter, Predicate linkedEntityFilter, + Supplier msg, FutureCallback callback) { boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); if (send) { clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); @@ -827,7 +829,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return msg.build(); } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request) { + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List result) { + //TODO: IM Use result in both methods to update the versions of telemetry/attributes. ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 4eb3f0cd89..19d6e295b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -62,9 +62,9 @@ import java.util.UUID; @Slf4j public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService implements TbCalculatedFieldConsumerService { - @Value("${queue.calculated_fields.poll_interval}") + @Value("${queue.calculated_fields.poll_interval:25}") private long pollInterval; - @Value("${queue.calculated_fields.pack_processing_timeout}") + @Value("${queue.calculated_fields.pack_processing_timeout:60000}") private long packProcessingTimeout; @Value("${queue.calculated_fields.consumer_per_partition:true}") private boolean consumerPerPartition; @@ -102,7 +102,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer this.calculatedFieldsExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(poolSize, "tb-cf-executor")); // TODO: multiple threads. this.mainConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() - .queueKey(new QueueKey(ServiceType.TB_CORE)) + .queueKey(new QueueKey(ServiceType.TB_RULE_ENGINE)) .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index b510b813d6..3064277bf4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -102,6 +102,7 @@ import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -570,7 +571,9 @@ public class DefaultTbClusterService implements TbClusterService { private void broadcast(ComponentLifecycleMsg msg) { ComponentLifecycleMsgProto componentLifecycleMsgProto = toProto(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); + TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); EntityType entityType = msg.getEntityId().getEntityType(); if (entityType.equals(EntityType.TENANT) || entityType.equals(EntityType.TENANT_PROFILE) @@ -581,7 +584,6 @@ public class DefaultTbClusterService implements TbClusterService { || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) || entityType.equals(EntityType.ENTITY_VIEW) || entityType.equals(EntityType.NOTIFICATION_RULE) - || entityType.equals(EntityType.CALCULATED_FIELD) ) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); @@ -594,6 +596,14 @@ public class DefaultTbClusterService implements TbClusterService { // No need to push notifications twice tbRuleEngineServices.removeAll(tbCoreServices); } + if (entityType.equals(EntityType.CALCULATED_FIELD)) { + for (String serviceId : tbCalculatedFieldServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); + toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCfNotificationMsg), null); + toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } + } for (String serviceId : tbRuleEngineServices) { TopicPartitionInfo tpi = topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); @@ -641,11 +651,11 @@ public class DefaultTbClusterService implements TbClusterService { if (deviceNameChanged) { gatewayNotificationsService.onDeviceUpdated(device, old); } - boolean deviceTypeChanged = !device.getType().equals(old.getType()); - if (deviceTypeChanged) { + boolean deviceProfileChanged = !device.getDeviceProfileId().equals(old.getDeviceProfileId()); + if (deviceProfileChanged) { handleEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } - if (deviceNameChanged || deviceTypeChanged) { + if (deviceNameChanged || deviceProfileChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { @@ -802,37 +812,13 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { var created = oldCalculatedField == null; - broadcastEntityChangeToTransport(calculatedField.getTenantId(), calculatedField.getId(), calculatedField, null); broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - sendCalculatedFieldEvent(calculatedField.getTenantId(), calculatedField.getId(), created, !created, false); } @Override public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { CalculatedFieldId calculatedFieldId = calculatedField.getId(); - broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); - sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); - } - - private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { - ComponentLifecycleMsgProto.Builder builder = ComponentLifecycleMsgProto.newBuilder(); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setEntityType(TransportProtos.EntityTypeProto.CALCULATED_FIELD); - builder.setEntityIdMSB(calculatedFieldId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); - TransportProtos.ComponentLifecycleEvent event; - if (added) { - event = TransportProtos.ComponentLifecycleEvent.CREATED; - } else if (updated) { - event = TransportProtos.ComponentLifecycleEvent.UPDATED; - } else { - event = TransportProtos.ComponentLifecycleEvent.DELETED; - } - builder.setEvent(event); - - pushNotificationToCalculatedFields(tenantId, calculatedFieldId, ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(builder).build(), null); } private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 0f13dbb2f9..7f889d431f 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -167,9 +167,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer public void saveAttributesInternal(AttributesSaveRequest request) { log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); - addMainCallback(saveFuture, request.getCallback()); DonAsynchron.withCallback(saveFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request); + calculatedFieldExecutionService.pushRequestToQueue(request, result); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index fdd9923292..d62aac5e6c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1750,7 +1750,7 @@ queue: # Amount of partitions used by CF microservices partitions: "${TB_QUEUE_CF_PARTITIONS:10}" # Timeout for processing a message pack by CF microservices - pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:60000}" # Enable/disable a separate consumer per partition for CF queue consumer_per_partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" # Thread pool size for processing of the incoming messages From bd34ed5011ac6d854bd06dcd92095662bc1a16b0 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 15:34:18 +0200 Subject: [PATCH 126/416] Implemented calculated fields table --- .../core/http/calculated-fields.service.ts | 78 +++++++++ .../calculated-fields-table-config.ts | 157 ++++++++++++++++++ .../calculated-fields-table.component.html | 1 + .../calculated-fields-table.component.ts | 104 ++++++++++++ .../entity/entities-table.component.ts | 8 +- .../home/components/home-components.module.ts | 9 +- .../entity/entity-table-component.models.ts | 4 +- .../pages/device/device-tabs.component.html | 4 + .../app/shared/models/entity-type.models.ts | 15 +- .../assets/locale/locale.constant-en_US.json | 9 + 10 files changed, 383 insertions(+), 6 deletions(-) create mode 100644 ui-ngx/src/app/core/http/calculated-fields.service.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts new file mode 100644 index 0000000000..92eae2c25a --- /dev/null +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -0,0 +1,78 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; +import { Observable, of } from 'rxjs'; +import { HttpClient } from '@angular/common/http'; +import { PageData } from '@shared/models/page/page-data'; + +@Injectable({ + providedIn: 'root' +}) +// [TODO]: [Calculated fields] - implement when BE ready +export class CalculatedFieldsService { + + fieldsMock = [ + { + name: 'Calculated Field 1', + type: 'Simple', + expression: '1 + 2', + id: { + id: '1', + } + }, + { + name: 'Calculated Field 2', + type: 'Script', + expression: '${power}', + id: { + id: '2', + } + } + ]; + + constructor( + private http: HttpClient + ) { } + + public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + return of(this.fieldsMock[0]); + // return this.http.get(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + } + + public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { + return of(this.fieldsMock[1]); + // return this.http.post('/api/calculated-field', calculatedField, defaultHttpOptionsFromConfig(config)); + } + + public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + return of(true); + // return this.http.delete(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + } + + public getCalculatedFields(query: any, + config?: RequestConfig): Observable> { + return of({ + data: this.fieldsMock, + totalPages: 1, + totalElements: 2, + hasNext: false, + }); + // return this.http.get>(`/api/calculated-field${query.toQuery()}`, + // defaultHttpOptionsFromConfig(config)); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts new file mode 100644 index 0000000000..f2e2a64b49 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -0,0 +1,157 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityTableColumn, EntityTableConfig } from '@home/models/entity/entities-table-config.models'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { TranslateService } from '@ngx-translate/core'; +import { Direction } from '@shared/models/page/sort-order'; +import { MatDialog } from '@angular/material/dialog'; +import { TimePageLink } from '@shared/models/page/page-link'; +import { Observable, of } from 'rxjs'; +import { PageData } from '@shared/models/page/page-data'; +import { EntityId } from '@shared/models/id/entity-id'; +import { DialogService } from '@core/services/dialog.service'; +import { MINUTE } from '@shared/models/time/time.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { ChangeDetectorRef, DestroyRef, ViewContainerRef } from '@angular/core'; +import { Overlay } from '@angular/cdk/overlay'; +import { UtilsService } from '@core/services/utils.service'; +import { EntityService } from '@core/http/entity.service'; +import { EntityDebugSettings } from '@shared/models/entity.models'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { catchError, switchMap } from 'rxjs/operators'; + +export class CalculatedFieldsTableConfig extends EntityTableConfig { + + readonly calculatedFieldsDebugPerTenantLimitsConfiguration = + getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; + readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; + + constructor(private calculatedFieldsService: CalculatedFieldsService, + private entityService: EntityService, + private dialogService: DialogService, + private translate: TranslateService, + private dialog: MatDialog, + public entityId: EntityId = null, + private store: Store, + private viewContainerRef: ViewContainerRef, + private overlay: Overlay, + private cd: ChangeDetectorRef, + private utilsService: UtilsService, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, + private destroyRef: DestroyRef, + ) { + super(); + this.tableTitle = this.translate.instant('calculated-fields.label'); + this.detailsPanelEnabled = false; + this.selectionEnabled = true; + this.searchEnabled = true; + this.addEnabled = true; + this.entitiesDeleteEnabled = true; + this.actionsColumnTitle = ''; + this.entityType = EntityType.CALCULATED_FIELDS; + this.entityTranslations = entityTypeTranslations.get(EntityType.CALCULATED_FIELDS); + + this.entitiesFetchFunction = pageLink => this.fetchCalculatedFields(pageLink); + + this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; + + this.columns.push( + new EntityTableColumn('name', 'common.name', '33%')); + this.columns.push( + new EntityTableColumn('type', 'common.type', '50px')); + this.columns.push( + new EntityTableColumn('expression', 'calculated-fields.expression', '50%')); + + this.cellActionDescriptors.push( + { + name: '', + nameFunction: (entity) => this.getDebugConfigLabel(entity?.debugSettings), + icon: 'mdi:bug', + isEnabled: () => true, + iconFunction: ({ debugSettings }) => this.isDebugActive(debugSettings?.allEnabledUntil) || debugSettings?.failuresEnabled ? 'mdi:bug' : 'mdi:bug-outline', + onAction: ($event, entity) => this.onOpenDebugConfig($event, entity), + }, + { + name: this.translate.instant('action.edit'), + icon: 'edit', + isEnabled: () => true, + // // [TODO]: [Calculated fields] - implement edit + onAction: (_, entity) => {} + } + ); + } + + fetchCalculatedFields(pageLink: TimePageLink): Observable> { + return this.calculatedFieldsService.getCalculatedFields(pageLink); + } + + onOpenDebugConfig($event: Event, { debugSettings = {}, id }: any): void { + const { renderer, viewContainerRef } = this.getTable(); + if ($event) { + $event.stopPropagation(); + } + const trigger = $event.target as Element; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const debugStrategyPopover = this.popoverService.displayPopover(trigger, renderer, + viewContainerRef, EntityDebugSettingsPanelComponent, 'bottom', true, null, + { + debugLimitsConfiguration: this.calculatedFieldsDebugPerTenantLimitsConfiguration, + maxDebugModeDuration: this.maxDebugModeDuration, + entityLabel: this.translate.instant('debug-settings.integration'), + ...debugSettings + }, + {}, + {}, {}, true); + debugStrategyPopover.tbComponentRef.instance.popover = debugStrategyPopover; + debugStrategyPopover.tbComponentRef.instance.onSettingsApplied.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((settings: EntityDebugSettings) => { + this.onDebugConfigChanged(id.id, settings); + debugStrategyPopover.hide(); + }); + } + } + + private getDebugConfigLabel(debugSettings: EntityDebugSettings): string { + const isDebugActive = this.isDebugActive(debugSettings?.allEnabledUntil); + + if (!isDebugActive) { + return debugSettings?.failuresEnabled ? this.translate.instant('debug-settings.failures') : this.translate.instant('common.disabled'); + } else { + return this.durationLeft.transform(debugSettings?.allEnabledUntil) + } + } + + private isDebugActive(allEnabledUntil: number): boolean { + return allEnabledUntil > new Date().getTime(); + } + + private onDebugConfigChanged(id: string, debugSettings: EntityDebugSettings): void { + this.calculatedFieldsService.getCalculatedField(id).pipe( + switchMap(field => this.calculatedFieldsService.saveCalculatedField({ ...field, debugSettings })), + catchError(() => of(null)), + takeUntilDestroyed(this.destroyRef), + ).subscribe(() => this.updateData()); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html new file mode 100644 index 0000000000..38aa1487af --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html @@ -0,0 +1 @@ + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts new file mode 100644 index 0000000000..a5b1ab34c1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -0,0 +1,104 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + DestroyRef, + Input, + OnInit, + ViewChild, + ViewContainerRef, +} from '@angular/core'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntitiesTableComponent } from '@home/components/entity/entities-table.component'; +import { EntityService } from '@core/http/entity.service'; +import { DialogService } from '@core/services/dialog.service'; +import { TranslateService } from '@ngx-translate/core'; +import { MatDialog } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Overlay } from '@angular/cdk/overlay'; +import { UtilsService } from '@core/services/utils.service'; +import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/calculated-fields-table-config'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; + +@Component({ + selector: 'tb-calculated-fields-table', + templateUrl: './calculated-fields-table.component.html', + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class CalculatedFieldsTableComponent implements OnInit { + + @Input() entityId: EntityId; + + @Input() + set active(active: boolean) { + if (this.activeValue !== active) { + this.activeValue = active; + if (this.activeValue && this.dirtyValue) { + this.dirtyValue = false; + this.entitiesTable.updateData(); + } + } + } + + @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; + + calculatedFieldsTableConfig: CalculatedFieldsTableConfig; + + private activeValue = false; + private dirtyValue = false; + + constructor(private calculatedFieldsService: CalculatedFieldsService, + private entityService: EntityService, + private dialogService: DialogService, + private translate: TranslateService, + private dialog: MatDialog, + private store: Store, + private overlay: Overlay, + private viewContainerRef: ViewContainerRef, + private cd: ChangeDetectorRef, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, + private destroyRef: DestroyRef, + private utilsService: UtilsService) { + } + + ngOnInit() { + this.dirtyValue = !this.activeValue; + + this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( + this.calculatedFieldsService, + this.entityService, + this.dialogService, + this.translate, + this.dialog, + this.entityId, + this.store, + this.viewContainerRef, + this.overlay, + this.cd, + this.utilsService, + this.durationLeft, + this.popoverService, + this.destroyRef + ); + } +} diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index 9bb5fe4d8a..3c723f8080 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -25,8 +25,10 @@ import { OnChanges, OnDestroy, OnInit, + Renderer2, SimpleChanges, - ViewChild + ViewChild, + ViewContainerRef, } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; @@ -141,7 +143,9 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa private router: Router, private elementRef: ElementRef, private fb: FormBuilder, - private zone: NgZone) { + private zone: NgZone, + public viewContainerRef: ViewContainerRef, + public renderer: Renderer2) { super(store); } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 32e509e842..1a6b9c08e0 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -183,6 +183,8 @@ import { } from '@home/components/dashboard-page/layout/select-dashboard-breakpoint.component'; import { EntityChipsComponent } from '@home/components/entity/entity-chips.component'; import { DashboardViewComponent } from '@home/components/dashboard-view/dashboard-view.component'; +import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; @NgModule({ declarations: @@ -326,7 +328,8 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar RateLimitsDetailsDialogComponent, SendNotificationButtonComponent, EntityChipsComponent, - DashboardViewComponent + DashboardViewComponent, + CalculatedFieldsTableComponent, ], imports: [ CommonModule, @@ -463,11 +466,13 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar RateLimitsDetailsDialogComponent, SendNotificationButtonComponent, EntityChipsComponent, - DashboardViewComponent + DashboardViewComponent, + CalculatedFieldsTableComponent, ], providers: [ WidgetComponentService, CustomDialogService, + DurationLeftPipe, {provide: EMBED_DASHBOARD_DIALOG_TOKEN, useValue: EmbedDashboardDialogComponent}, {provide: COMPLEX_FILTER_PREDICATE_DIALOG_COMPONENT_TOKEN, useValue: ComplexFilterPredicateDialogComponent}, {provide: DASHBOARD_PAGE_COMPONENT_TOKEN, useValue: DashboardPageComponent}, diff --git a/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts b/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts index b6f634195e..8c4f856609 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts @@ -20,7 +20,7 @@ import { SafeHtml } from '@angular/platform-browser'; import { PageLink } from '@shared/models/page/page-link'; import { Timewindow } from '@shared/models/time/time.models'; import { EntitiesDataSource } from '@home/models/datasource/entity-datasource'; -import { ElementRef, EventEmitter } from '@angular/core'; +import { ElementRef, EventEmitter, Renderer2, ViewContainerRef } from '@angular/core'; import { TbAnchorComponent } from '@shared/components/tb-anchor.component'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -64,6 +64,8 @@ export interface IEntitiesTableComponent { paginator: MatPaginator; sort: MatSort; route: ActivatedRoute; + viewContainerRef: ViewContainerRef; + renderer: Renderer2; addEnabled(): boolean; clearSelection(): void; diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index 5e30694719..357bb587cc 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -32,6 +32,10 @@ [entityName]="entity.name"> + + + diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index e1b59a9243..5b540a6c54 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -49,7 +49,8 @@ export enum EntityType { OAUTH2_CLIENT = 'OAUTH2_CLIENT', DOMAIN = 'DOMAIN', MOBILE_APP_BUNDLE = 'MOBILE_APP_BUNDLE', - MOBILE_APP = 'MOBILE_APP' + MOBILE_APP = 'MOBILE_APP', + CALCULATED_FIELDS = 'CALCULATED_FIELDS', } export enum AliasEntityType { @@ -478,6 +479,18 @@ export const entityTypeTranslations = new MapAre you sure you want to leave this page?", @@ -1027,6 +1034,8 @@ "city-max-length": "Specified city should be less than 256" }, "common": { + "name": "Name", + "type": "Type", "username": "Username", "password": "Password", "enter-username": "Enter username", From a652b31d7f880b1c13734c26c5f331e971a524e5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 24 Jan 2025 15:41:00 +0200 Subject: [PATCH 127/416] implementation processing notification --- .../service/cf/CalculatedFieldCache.java | 5 +- .../cf/CalculatedFieldExecutionService.java | 7 +- .../cf/DefaultCalculatedFieldCache.java | 32 ++++ ...efaultCalculatedFieldExecutionService.java | 164 +++++++++++------- .../entitiy/EntityStateSourcingListener.java | 21 +-- .../DefaultSystemDataLoaderService.java | 7 +- ...faultTbCalculatedFieldConsumerService.java | 2 +- .../queue/DefaultTbClusterService.java | 74 ++++---- .../queue/DefaultTbCoreConsumerService.java | 41 ----- .../processing/AbstractConsumerService.java | 4 + .../TbRuleEngineQueueConsumerManager.java | 14 +- .../DefaultTelemetrySubscriptionService.java | 2 - .../server/cluster/TbClusterService.java | 8 +- .../server/common/util/ProtoUtils.java | 15 ++ common/proto/src/main/proto/queue.proto | 4 - .../server/dao/service/EntityServiceTest.java | 5 +- 16 files changed, 211 insertions(+), 194 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index ea55894432..1ee1d4d562 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -42,6 +41,10 @@ public interface CalculatedFieldCache { Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + void evictProfile(TenantId tenantId, EntityId entityId); + + void evictEntity(TenantId tenantId, EntityId entityId); + void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 93a0ecc75f..836af03e5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -20,8 +20,9 @@ import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; import java.util.List; @@ -42,9 +43,9 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); - void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); + void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback); -// void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); + void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback); void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 7e841a0cf8..4278e74845 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -22,11 +22,16 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -172,6 +177,33 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entities; } + @Override + public void evictProfile(TenantId tenantId, EntityId entityId) { + log.debug("[{}] evict entity profile from cache.", entityId); + profileEntities.remove(entityId); + } + + @Override + public void evictEntity(TenantId tenantId, EntityId entityId) { + calculatedFieldFetchLock.lock(); + try { + profileEntities.forEach((profile, entityIds) -> entityIds.remove(entityId)); + if (EntityType.ASSET.equals(entityId.getEntityType())) { + Asset asset = assetService.findAssetById(tenantId, (AssetId) entityId); + if (asset != null) { + profileEntities.computeIfAbsent(asset.getAssetProfileId(), profileId -> new HashSet<>()).add(entityId); + } + } else { + Device device = deviceService.findDeviceById(tenantId, (DeviceId) entityId); + if (device != null) { + profileEntities.computeIfAbsent(device.getDeviceProfileId(), profileId -> new HashSet<>()).add(entityId); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { calculatedFieldFetchLock.lock(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d46a7515ef..beb347d29f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,6 +36,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -72,14 +73,17 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; @@ -92,7 +96,9 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -334,7 +340,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas initializeStateForEntity(calculatedFieldCtx, entityId, callback); } case ASSET_PROFILE, DEVICE_PROFILE -> { - log.info("Initializing state for all entities in profile: tenantICalculatedFieldMsgProtod=[{}], profileId=[{}]", tenantId, entityId); + log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> entry.getValue().getRefEntityId() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -401,8 +407,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest request) { + public void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback) { try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); EntityId entityId = request.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { @@ -418,15 +425,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas processCalculatedFieldLinks(request, tpiStatesToUpdate); if (!tpiStatesToUpdate.isEmpty()) { tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request, ctxIds); - clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() - .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(proto, ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); }); } } else { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request); - clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() - .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(proto).build(), null); } } } catch (Exception e) { @@ -479,30 +483,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } -// @Override -// public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { -// try { -// CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); -// -// if (proto.getLinksList().isEmpty()) { -// onTelemetryUpdate(request); -// return; -// } -// -// proto.getLinksList().forEach(ctxIdProto -> { -// CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); -// CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); -// -// Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); -// if (!updatedTelemetry.isEmpty()) { -// EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); -// executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); -// } -// }); -// } catch (Exception e) { -// log.trace("Failed to process telemetry update msg: [{}]", proto, e); -// } -// } + @Override + public void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback) { + try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(proto.getMsg()); + + if (proto.getLinksList().isEmpty()) { + onTelemetryUpdate(proto, callback); + return; + } + + proto.getLinksList().forEach(ctxIdProto -> { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); + + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); + if (!updatedTelemetry.isEmpty()) { + EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } + }); + } catch (Exception e) { + log.trace("Failed to process telemetry update msg: [{}]", proto, e); + } + } private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); @@ -753,23 +757,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } -// private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { -// return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() -// .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) -// .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) -// .setEntityType(ctxId.entityId().getEntityType().name()) -// .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) -// .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) -// .build(); -// } -// -// private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { -// return TransportProtos.CalculatedFieldIdProto.newBuilder() -// .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) -// .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) -// .build(); -// } - private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); @@ -821,22 +808,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); - for (TsKvEntry entry : request.getEntries()) { - telemetryMsg.addTsData(ProtoUtils.toTsKvProto(entry)); + List entries = request.getEntries(); + List versions = result.getVersions(); + for (int i = 0; i < entries.size(); i++) { + long tsVersion = versions.get(i); + TsKvProto tsProto = ProtoUtils.toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); + telemetryMsg.addTsData(tsProto); } msg.setTelemetryMsg(telemetryMsg.build()); return msg.build(); } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List result) { - //TODO: IM Use result in both methods to update the versions of telemetry/attributes. + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); - for (AttributeKvEntry entry : request.getEntries()) { - telemetryMsg.addAttrData(ProtoUtils.toProto(entry)); + List entries = request.getEntries(); + for (int i = 0; i < entries.size(); i++) { + long attrVersion = versions.get(i); + AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); + telemetryMsg.addAttrData(attrProto); } msg.setTelemetryMsg(telemetryMsg.build()); @@ -854,15 +847,66 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); for (CalculatedFieldId cfId : calculatedFieldIds) { - CalculatedFieldIdProto.Builder calculatedFieldIdProto = CalculatedFieldIdProto.newBuilder(); - calculatedFieldIdProto.setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()); - calculatedFieldIdProto.setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()); - telemetryMsg.addPreviousCalculatedFields(calculatedFieldIdProto.build()); + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); } return telemetryMsg; } + private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { + TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + builder.setMsg(telemetryProto); + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + return builder.build(); + } + + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + + private CalculatedFieldTelemetryUpdateRequest fromProto(CalculatedFieldTelemetryMsgProto proto) { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + } else { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, scope, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + } + } + private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index ab8246ccf5..4703ed1606 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -88,7 +87,7 @@ public class EntityStateSourcingListener { case ASSET -> { onAssetUpdate(event.getEntity(), event.getOldEntity()); } - case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE, CALCULATED_FIELD -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -123,9 +122,6 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } - case CALCULATED_FIELD -> { - onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); - } default -> { } } @@ -150,7 +146,7 @@ public class EntityStateSourcingListener { Asset asset = (Asset) event.getEntity(); tbClusterService.onAssetDeleted(tenantId, asset, null); } - case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE, CALCULATED_FIELD -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { @@ -191,10 +187,6 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } - case CALCULATED_FIELD -> { - CalculatedField calculatedField = (CalculatedField) event.getEntity(); - tbClusterService.onCalculatedFieldDeleted(tenantId, calculatedField, null); - } default -> { } } @@ -275,15 +267,6 @@ public class EntityStateSourcingListener { } } - private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { - CalculatedField calculatedField = (CalculatedField) entity; - CalculatedField oldCalculatedField = null; - if (oldEntity instanceof CalculatedField) { - oldCalculatedField = (CalculatedField) oldEntity; - } - tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField); - } - private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index fd8a59c392..5b0c790215 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -69,6 +69,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageLink; @@ -98,9 +99,9 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.notification.NotificationSettingsService; import org.thingsboard.server.dao.notification.NotificationTargetService; -import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; @@ -308,7 +309,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { jwtSettingsService.saveJwtSettings(jwtSettings); } - List mobiles = mobileAppDao.findByTenantId(TenantId.SYS_TENANT_ID, null, new PageLink(Integer.MAX_VALUE,0)).getData(); + List mobiles = mobileAppDao.findByTenantId(TenantId.SYS_TENANT_ID, null, new PageLink(Integer.MAX_VALUE, 0)).getData(); if (CollectionUtils.isNotEmpty(mobiles)) { mobiles.stream() .filter(mobileApp -> !validateKeyLength(mobileApp.getAppSecret())) @@ -571,7 +572,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { private void save(DeviceId deviceId, String key, boolean value) { if (persistActivityToTelemetry) { - ListenableFuture saveFuture = tsService.save( + ListenableFuture saveFuture = tsService.save( TenantId.SYS_TENANT_ID, deviceId, Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), 0L); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 19d6e295b5..3df2653df6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -171,7 +171,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer callback.onSuccess(); } -// private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { + // private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { // var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); // var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); // var oldProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getOldProfileIdMSB(), profileUpdateMsg.getOldProfileIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 3064277bf4..04d6a53401 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,12 +38,10 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -347,6 +345,13 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } + @Override public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); @@ -409,7 +414,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - handleProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); + handleCalculatedFieldEntityDeleted(tenantId, deviceId, device.getDeviceProfileId()); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -418,7 +423,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - handleProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), true, true); + handleCalculatedFieldEntityDeleted(tenantId, assetId, asset.getAssetProfileId()); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } @@ -653,13 +658,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceProfileChanged = !device.getDeviceProfileId().equals(old.getDeviceProfileId()); if (deviceProfileChanged) { - handleEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + handleCalculatedFieldEntityUpdated(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceProfileChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { - handleProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); + handleCalculatedFieldEntityAdded(device.getTenantId(), device.getId(), device.getDeviceProfileId()); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -673,10 +678,10 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + handleCalculatedFieldEntityUpdated(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } } else { - handleProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); + handleCalculatedFieldEntityAdded(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @@ -809,52 +814,41 @@ public class DefaultTbClusterService implements TbClusterService { } } - @Override - public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { - var created = oldCalculatedField == null; - broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + private void handleCalculatedFieldEntityAdded(TenantId tenantId, EntityId entityId, EntityId newProfileId) { + handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, null, newProfileId, true, false, false); } - @Override - public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { - CalculatedFieldId calculatedFieldId = calculatedField.getId(); - broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + private void handleCalculatedFieldEntityUpdated(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, newProfileId, false, true, true); } - private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - builder.setEntityProfileType(newProfileId.getEntityType().name()); - builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); - builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); - builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); - builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); - TransportProtos.EntityProfileUpdateMsgProto msg = builder.build(); - - broadcastToCore(ToCoreNotificationMsg.newBuilder().setEntityProfileUpdateMsg(msg).build()); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); + private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId, EntityId oldProfileId) { + handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, null, false, false, true); } - private void handleProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { - TransportProtos.ProfileEntityMsgProto.Builder builder = TransportProtos.ProfileEntityMsgProto.newBuilder(); + private void handleCalculatedFieldEntityUpdateEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId, boolean added, boolean updated, boolean deleted) { + TransportProtos.CalculatedFieldEntityUpdateMsgProto.Builder builder = TransportProtos.CalculatedFieldEntityUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setEntityType(entityId.getEntityType().name()); builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - builder.setEntityProfileType(profileId.getEntityType().name()); - builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); - builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); + if (oldProfileId != null) { + builder.setEntityProfileType(oldProfileId.getEntityType().name()); + builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); + } + if (newProfileId != null) { + builder.setEntityProfileType(newProfileId.getEntityType().name()); + builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); + builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); + } builder.setAdded(added); + builder.setUpdated(updated); builder.setDeleted(deleted); - TransportProtos.ProfileEntityMsgProto msg = builder.build(); + TransportProtos.CalculatedFieldEntityUpdateMsgProto msg = builder.build(); - broadcastToCore(ToCoreNotificationMsg.newBuilder().setProfileEntityMsg(msg).build()); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setProfileEntityMsg(msg).build(), null); + pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index e3a1ca22d3..e7bde845f6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -39,8 +39,6 @@ import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -88,7 +86,6 @@ import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -110,7 +107,6 @@ import org.thingsboard.server.service.ws.notification.sub.NotificationRequestUpd import org.thingsboard.server.service.ws.notification.sub.NotificationUpdate; import java.util.List; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -153,14 +149,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; private QueueConsumerManager> firmwareStatesConsumer; private volatile ListeningExecutorService deviceActivityEventsExecutor; - private volatile ListeningExecutorService calculatedFieldsExecutor; public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, @@ -183,7 +177,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) @@ -319,12 +310,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); - if (added) { - entitiesByProfile.add(entityId); - } else { - entitiesByProfile.remove(entityId); - } - } - private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { if (msg.hasSubEvent()) { TbEntitySubEventProto subEvent = msg.getSubEvent(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index dac35bfc5c..5b1e5d7d79 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -178,12 +178,16 @@ public abstract class AbstractConsumerService onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } @Override diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index b28bf73c89..d2fbcddfdf 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; @@ -40,6 +39,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -79,6 +79,8 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback); + void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback); void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); @@ -125,8 +127,4 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); - void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField); - - void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback); - } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 3bace4d91f..6a40f13688 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -58,6 +58,7 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; @@ -628,6 +629,20 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static TsKvEntry fromProto(TransportProtos.TsKvProto proto) { + TransportProtos.KeyValueProto kvProto = proto.getKv(); + String key = kvProto.getKey(); + KvEntry entry = switch (kvProto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); + case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); + case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); + default -> null; + }; + return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); + } + public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { return TransportProtos.TsKvProto.newBuilder() .setTs(tsKvEntry.getTs()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 288c923aaa..ede796b12b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1591,8 +1591,6 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; -// CalculatedFieldMsgProto calculatedFieldMsg = 53; -// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1612,8 +1610,6 @@ message ToCoreNotificationMsg { FromEdgeSyncResponseMsgProto fromEdgeSyncResponse = 12 [deprecated = true]; ResourceCacheInvalidateMsg resourceCacheInvalidateMsg = 13; RestApiCallResponseMsgProto restApiCallResponseMsg = 50; -// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 51; -// ProfileEntityMsgProto profileEntityMsg = 52; } /* Messages to Edge queue that are handled by ThingsBoard Core Service */ diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java index a26f345fb7..8c93245837 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java @@ -46,6 +46,7 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; @@ -1824,7 +1825,7 @@ public class EntityServiceTest extends AbstractServiceTest { } } - List> timeseriesFutures = new ArrayList<>(); + List> timeseriesFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i))); @@ -2430,7 +2431,7 @@ public class EntityServiceTest extends AbstractServiceTest { return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { + private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { TsKvEntity tsKv = new TsKvEntity(); tsKv.setStrKey(key); tsKv.setDoubleValue(value); From c332e7373f507199552a814c424dd98b07dd4233 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Jan 2025 15:45:00 +0200 Subject: [PATCH 128/416] WIP: CalculatedFieldConsumer refactoring --- .../cf/CalculatedFieldExecutionService.java | 6 ++ ...efaultCalculatedFieldExecutionService.java | 13 +++ ...faultTbCalculatedFieldConsumerService.java | 85 ++++++++++++++++++- .../queue/DefaultTbCoreConsumerService.java | 13 +-- .../queue/DefaultTbEdgeConsumerService.java | 13 +-- .../service/queue/PendingMsgHolder.java | 24 ++++++ 6 files changed, 133 insertions(+), 21 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 93a0ecc75f..893dba92a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -20,6 +20,8 @@ import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; @@ -36,6 +38,10 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result); + void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback); + + void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback); + // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d46a7515ef..2638f22254 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -75,6 +75,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -234,6 +235,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return send; } + @Override + public void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + + callback.onSuccess(); + } + + @Override + public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + + callback.onSuccess(); + } + @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 19d6e295b5..b36b34dd94 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,6 +21,8 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -32,12 +34,19 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -50,12 +59,20 @@ import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.queue.DefaultTbCoreConsumerService.PendingMsgHolder; import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import java.util.List; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Service @TbRuleEngineComponent @@ -132,7 +149,46 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer } private void processMsgs(List> msgs, TbQueueConsumer> consumer, CalculatedFieldQueueConfig config) throws Exception { - + List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); + ConcurrentMap> pendingMap = orderedMsgList.stream().collect( + Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); + Future packSubmitFuture = consumersExecutor.submit(() -> { + orderedMsgList.forEach((element) -> { + UUID id = element.getUuid(); + TbProtoQueueMsg msg = element.getMsg(); + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + ToCalculatedFieldMsg toCfMsg = msg.getValue(); + pendingMsgHolder.setMsg(toCfMsg); + if (toCfMsg.hasTelemetryMsg()) { + log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); + forwardToCalculatedFieldService(toCfMsg.getTelemetryMsg(), callback); + } else if (toCfMsg.hasLinkedTelemetryMsg()) { + log.trace("[{}] Forwarding linked telemetry message for processing {}", id, toCfMsg.getLinkedTelemetryMsg()); + forwardToCalculatedFieldService(toCfMsg.getLinkedTelemetryMsg(), callback); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); + } + }); + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!packSubmitFuture.isDone()) { + packSubmitFuture.cancel(true); + log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); + } + if (log.isDebugEnabled()) { + ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); + } + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + } + consumer.commit(); } @Override @@ -193,6 +249,33 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer // } // } // + + private void forwardToCalculatedFieldService(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + var msg = linkedMsg.getMsg(); + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onLinkedTelemetryMsg(linkedMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + + } + + private void forwardToCalculatedFieldService(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onTelemetryMsg(msg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index e3a1ca22d3..ca42298742 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -269,7 +269,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future packSubmitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { UUID id = element.getUuid(); @@ -278,7 +278,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService(id, ctx); try { ToCoreMsg toCoreMsg = msg.getValue(); - pendingMsgHolder.setToCoreMsg(toCoreMsg); + pendingMsgHolder.setMsg(toCoreMsg); if (toCoreMsg.hasToSubscriptionMgrMsg()) { log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); @@ -335,8 +335,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); @@ -346,12 +345,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future submitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { UUID id = element.getUuid(); @@ -145,7 +145,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService(id, ctx); try { ToEdgeMsg toEdgeMsg = msg.getValue(); - pendingMsgHolder.setToEdgeMsg(toEdgeMsg); + pendingMsgHolder.setMsg(toEdgeMsg); if (toEdgeMsg.hasEdgeNotificationMsg()) { pushNotificationToEdge(toEdgeMsg.getEdgeNotificationMsg(), 0, packProcessingRetries, callback); } @@ -161,20 +161,13 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); } - private static class PendingMsgHolder { - @Getter - @Setter - private volatile ToEdgeMsg toEdgeMsg; - } - @Override protected ServiceType getServiceType() { return ServiceType.TB_CORE; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java b/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java new file mode 100644 index 0000000000..8793e45da6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Getter; +import lombok.Setter; + +public class PendingMsgHolder { + @Getter @Setter + private volatile T msg; +} From b169dfab2793ef1b0721f1b4b4ab7545823bdf81 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 15:46:48 +0200 Subject: [PATCH 129/416] added license headers --- .../calculated-fields-table.component.html | 17 ++++++++++++++ .../calculated-fields-table.component.scss | 22 +++++++++++++++++++ .../calculated-fields-table.component.ts | 1 + 3 files changed, 40 insertions(+) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html index 38aa1487af..d627e16ce9 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html @@ -1 +1,18 @@ + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss new file mode 100644 index 0000000000..ea3f7d90b7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + tb-entities-table { + .mat-drawer-container { + background-color: white; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index a5b1ab34c1..5449bce5a7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -42,6 +42,7 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; @Component({ selector: 'tb-calculated-fields-table', templateUrl: './calculated-fields-table.component.html', + styleUrls: ['./calculated-fields-table.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush, }) export class CalculatedFieldsTableComponent implements OnInit { From be4ea19b91636d2e9c1e927a4776c954972ebc8c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 17:20:55 +0200 Subject: [PATCH 130/416] added calculated fields typing --- .../core/http/calculated-fields.service.ts | 21 ++++++--- .../calculated-fields-table-config.ts | 21 ++++----- .../pages/device/device-tabs.component.html | 2 +- .../shared/models/calculated-field.models.ts | 44 +++++++++++++++++++ .../app/shared/models/entity-type.models.ts | 8 ++-- .../shared/models/id/calculated-field-id.ts | 26 +++++++++++ .../assets/locale/locale.constant-en_US.json | 3 +- 7 files changed, 103 insertions(+), 22 deletions(-) create mode 100644 ui-ngx/src/app/shared/models/calculated-field.models.ts create mode 100644 ui-ngx/src/app/shared/models/id/calculated-field-id.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 92eae2c25a..66e0833128 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -19,6 +19,7 @@ import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; +import { CalculatedField } from '@shared/models/calculated-field.models'; @Injectable({ providedIn: 'root' @@ -30,7 +31,11 @@ export class CalculatedFieldsService { { name: 'Calculated Field 1', type: 'Simple', - expression: '1 + 2', + configuration: { + expression: '1 + 2', + type: 'SIMPLE', + }, + entityId: '1', id: { id: '1', } @@ -38,23 +43,27 @@ export class CalculatedFieldsService { { name: 'Calculated Field 2', type: 'Script', - expression: '${power}', + entityId: '2', + configuration: { + expression: '${power}', + type: 'SIMPLE', + }, id: { id: '2', } } - ]; + ] as any[]; constructor( private http: HttpClient ) { } - public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { return of(this.fieldsMock[0]); // return this.http.get(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { + public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { return of(this.fieldsMock[1]); // return this.http.post('/api/calculated-field', calculatedField, defaultHttpOptionsFromConfig(config)); } @@ -65,7 +74,7 @@ export class CalculatedFieldsService { } public getCalculatedFields(query: any, - config?: RequestConfig): Observable> { + config?: RequestConfig): Observable> { return of({ data: this.fieldsMock, totalPages: 1, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index f2e2a64b49..22f742b9d8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -39,8 +39,9 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, switchMap } from 'rxjs/operators'; +import { CalculatedField } from '@shared/models/calculated-field.models'; -export class CalculatedFieldsTableConfig extends EntityTableConfig { +export class CalculatedFieldsTableConfig extends EntityTableConfig { readonly calculatedFieldsDebugPerTenantLimitsConfiguration = getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; @@ -62,31 +63,31 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; this.columns.push( - new EntityTableColumn('name', 'common.name', '33%')); + new EntityTableColumn('name', 'common.name', '33%')); this.columns.push( - new EntityTableColumn('type', 'common.type', '50px')); + new EntityTableColumn('type', 'common.type', '50px')); this.columns.push( - new EntityTableColumn('expression', 'calculated-fields.expression', '50%')); + new EntityTableColumn('expression', 'calculated-fields.expression', '50%', entity => entity.configuration.expression)); this.cellActionDescriptors.push( { name: '', - nameFunction: (entity) => this.getDebugConfigLabel(entity?.debugSettings), + nameFunction: entity => this.getDebugConfigLabel(entity?.debugSettings), icon: 'mdi:bug', isEnabled: () => true, iconFunction: ({ debugSettings }) => this.isDebugActive(debugSettings?.allEnabledUntil) || debugSettings?.failuresEnabled ? 'mdi:bug' : 'mdi:bug-outline', @@ -102,11 +103,11 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { + fetchCalculatedFields(pageLink: TimePageLink): Observable> { return this.calculatedFieldsService.getCalculatedFields(pageLink); } - onOpenDebugConfig($event: Event, { debugSettings = {}, id }: any): void { + onOpenDebugConfig($event: Event, { debugSettings = {}, id }: CalculatedField): void { const { renderer, viewContainerRef } = this.getTable(); if ($event) { $event.stopPropagation(); diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index 357bb587cc..ab59f412cd 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -33,7 +33,7 @@ + label="{{ 'entity.type-calculated-fields' | translate }}" #calculatedFieldsTab="matTab"> , HasVersion, HasTenantId { + entityId: string; + type: CalculatedFieldType; + name: string; + debugSettings?: EntityDebugSettings; + externalId?: string; + createdTime?: number; + configuration: CalculatedFieldConfiguration; +} + +export enum CalculatedFieldType { + SIMPLE = 'SIMPLE', + COMPLEX = 'COMPLEX', +} + +export interface CalculatedFieldConfiguration { + type: CalculatedFieldConfigType; + expression: string; + arguments: Record; +} + +export enum CalculatedFieldConfigType { + SIMPLE = 'SIMPLE', + SCRIPT = 'SCRIPT', +} diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index 5b540a6c54..ee39be0f27 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -50,7 +50,7 @@ export enum EntityType { DOMAIN = 'DOMAIN', MOBILE_APP_BUNDLE = 'MOBILE_APP_BUNDLE', MOBILE_APP = 'MOBILE_APP', - CALCULATED_FIELDS = 'CALCULATED_FIELDS', + CALCULATED_FIELD = 'CALCULATED_FIELD', } export enum AliasEntityType { @@ -481,10 +481,10 @@ export const entityTypeTranslations = new Map Date: Fri, 24 Jan 2025 17:24:56 +0200 Subject: [PATCH 131/416] adjusted typing --- ui-ngx/src/app/core/http/calculated-fields.service.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 66e0833128..72c4f6c12c 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -20,6 +20,7 @@ import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; +import { PageLink } from '@shared/models/page/page-link'; @Injectable({ providedIn: 'root' @@ -73,7 +74,7 @@ export class CalculatedFieldsService { // return this.http.delete(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public getCalculatedFields(query: any, + public getCalculatedFields(pageLink: PageLink, config?: RequestConfig): Observable> { return of({ data: this.fieldsMock, @@ -81,7 +82,7 @@ export class CalculatedFieldsService { totalElements: 2, hasNext: false, }); - // return this.http.get>(`/api/calculated-field${query.toQuery()}`, + // return this.http.get>(`/api/calculated-field${pageLink.toQuery()}`, // defaultHttpOptionsFromConfig(config)); } } From 41b0963884edc21bb28569701d66fd269e7daa9e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 17:34:25 +0200 Subject: [PATCH 132/416] updated endpoint --- ui-ngx/src/app/core/http/calculated-fields.service.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 72c4f6c12c..5df4a84949 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -61,17 +61,17 @@ export class CalculatedFieldsService { public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { return of(this.fieldsMock[0]); - // return this.http.get(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + // return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { return of(this.fieldsMock[1]); - // return this.http.post('/api/calculated-field', calculatedField, defaultHttpOptionsFromConfig(config)); + // return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); } public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { return of(true); - // return this.http.delete(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + // return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } public getCalculatedFields(pageLink: PageLink, @@ -82,7 +82,7 @@ export class CalculatedFieldsService { totalElements: 2, hasNext: false, }); - // return this.http.get>(`/api/calculated-field${pageLink.toQuery()}`, + // return this.http.get>(`/api/calculatedField${pageLink.toQuery()}`, // defaultHttpOptionsFromConfig(config)); } } From ea7e6797edba6e113c581d5650870d7029d0af25 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 18:11:18 +0200 Subject: [PATCH 133/416] Implemented set entityId --- .../calculated-fields-table.component.ts | 22 ++++++++++++++----- .../shared/models/calculated-field.models.ts | 5 +---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 5449bce5a7..f98d24bd52 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -47,14 +47,23 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; }) export class CalculatedFieldsTableComponent implements OnInit { - @Input() entityId: EntityId; + @Input() + set entityId(entityId: EntityId) { + if (this.entityIdValue !== entityId) { + this.entityIdValue = entityId; + this.entitiesTable.resetSortAndFilter(this.activeValue); + if (!this.activeValue) { + this.hasInitialized = true; + } + } + } @Input() set active(active: boolean) { if (this.activeValue !== active) { this.activeValue = active; - if (this.activeValue && this.dirtyValue) { - this.dirtyValue = false; + if (this.activeValue && this.hasInitialized) { + this.hasInitialized = false; this.entitiesTable.updateData(); } } @@ -65,7 +74,8 @@ export class CalculatedFieldsTableComponent implements OnInit { calculatedFieldsTableConfig: CalculatedFieldsTableConfig; private activeValue = false; - private dirtyValue = false; + private hasInitialized = false; + private entityIdValue: EntityId; constructor(private calculatedFieldsService: CalculatedFieldsService, private entityService: EntityService, @@ -83,7 +93,7 @@ export class CalculatedFieldsTableComponent implements OnInit { } ngOnInit() { - this.dirtyValue = !this.activeValue; + this.hasInitialized = !this.activeValue; this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( this.calculatedFieldsService, @@ -91,7 +101,7 @@ export class CalculatedFieldsTableComponent implements OnInit { this.dialogService, this.translate, this.dialog, - this.entityId, + this.entityIdValue, this.store, this.viewContainerRef, this.overlay, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 714303e286..253bc58f39 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -17,13 +17,10 @@ import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/ent import { BaseData } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; -export interface CalculatedField extends BaseData, HasVersion, HasTenantId { - entityId: string; +export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { type: CalculatedFieldType; - name: string; debugSettings?: EntityDebugSettings; externalId?: string; - createdTime?: number; configuration: CalculatedFieldConfiguration; } From 0e1cd69e34645ce1acd3401d85d45ef5f6fe43a2 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 27 Jan 2025 15:55:40 +0200 Subject: [PATCH 134/416] updated cf consumer --- .../service/cf/CalculatedFieldCache.java | 4 - .../cf/CalculatedFieldExecutionService.java | 8 +- .../cf/DefaultCalculatedFieldCache.java | 32 ---- ...efaultCalculatedFieldExecutionService.java | 143 ++++++++++-------- .../entitiy/EntityStateSourcingListener.java | 22 ++- ...faultTbCalculatedFieldConsumerService.java | 63 ++++---- .../queue/DefaultTbClusterService.java | 47 +++++- .../processing/AbstractConsumerService.java | 4 - .../server/cluster/TbClusterService.java | 5 + .../server/common/util/ProtoUtils.java | 13 +- common/proto/src/main/proto/queue.proto | 2 + .../provider/TbCoreQueueProducerProvider.java | 3 +- 12 files changed, 188 insertions(+), 158 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 1ee1d4d562..ff3bda5da5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -41,10 +41,6 @@ public interface CalculatedFieldCache { Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); - void evictProfile(TenantId tenantId, EntityId entityId); - - void evictEntity(TenantId tenantId, EntityId entityId); - void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 62e234943b..7668acba4e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; @@ -30,6 +31,7 @@ public interface CalculatedFieldExecutionService { /** * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * * @param request - telemetry save request; * @param request - telemetry save result; */ @@ -37,6 +39,8 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result); + void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto); + void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback); void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback); @@ -47,10 +51,6 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); - void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback); - - void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback); - void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 4278e74845..7e841a0cf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -22,16 +22,11 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -177,33 +172,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entities; } - @Override - public void evictProfile(TenantId tenantId, EntityId entityId) { - log.debug("[{}] evict entity profile from cache.", entityId); - profileEntities.remove(entityId); - } - - @Override - public void evictEntity(TenantId tenantId, EntityId entityId) { - calculatedFieldFetchLock.lock(); - try { - profileEntities.forEach((profile, entityIds) -> entityIds.remove(entityId)); - if (EntityType.ASSET.equals(entityId.getEntityType())) { - Asset asset = assetService.findAssetById(tenantId, (AssetId) entityId); - if (asset != null) { - profileEntities.computeIfAbsent(asset.getAssetProfileId(), profileId -> new HashSet<>()).add(entityId); - } - } else { - Device device = deviceService.findDeviceById(tenantId, (DeviceId) entityId); - if (device != null) { - profileEntities.computeIfAbsent(device.getDeviceProfileId(), profileId -> new HashSet<>()).add(entityId); - } - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { calculatedFieldFetchLock.lock(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f92b9819ed..2a4d42ddea 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -66,7 +66,6 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; @@ -120,6 +119,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -242,14 +243,59 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(msg); + EntityId entityId = request.getEntityId(); + + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + TenantId tenantId = request.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + + if (tpi.isMyPartition()) { + + processCalculatedFields(request, entityId); + processCalculatedFields(request, getProfileId(tenantId, entityId)); - callback.onSuccess(); + Map> tpiStatesToUpdate = new HashMap<>(); + processCalculatedFieldLinks(request, tpiStatesToUpdate); + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg, ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); + }); + } + } else { + clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(msg).build(), null); + } + } + } catch (Exception e) { + log.trace("Failed to update telemetry.", e); + } } @Override public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(linkedMsg.getMsg()); + + if (linkedMsg.getLinksList().isEmpty()) { + onTelemetryMsg(linkedMsg.getMsg(), callback); + return; + } + + linkedMsg.getLinksList().forEach(ctxIdProto -> { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - callback.onSuccess(); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); + if (!updatedTelemetry.isEmpty()) { + EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } + }); + } catch (Exception e) { + log.trace("Failed to process telemetry update msg: [{}]", linkedMsg, e); + } } @Override @@ -263,7 +309,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Consumer resolvePartition = entityId -> { TopicPartitionInfo tpi; try { - tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); + tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); } @@ -378,6 +424,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + @Override + public void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto) { + EntityId entityId = calculatedField.getEntityId(); + ToCalculatedFieldMsg msg = ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(proto).build(); + switch (entityId.getEntityType()) { + case ASSET, DEVICE -> { + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null); + } + case ASSET_PROFILE, DEVICE_PROFILE -> { + Set tpiSet = calculatedFieldCache.getEntitiesByProfile(calculatedField.getTenantId(), entityId).stream() + .map(targetEntityId -> partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, targetEntityId)) + .collect(Collectors.toSet()); + tpiSet.forEach(tpi -> clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null)); + } + default -> throw new IllegalArgumentException("Entity type '" + calculatedField.getId().getEntityType() + + "' does not support calculated fields."); + } + } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getId()); boolean shouldReinit = true; @@ -418,38 +484,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged; } - @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); - EntityId entityId = request.getEntityId(); - - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - TenantId tenantId = request.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - - if (tpi.isMyPartition()) { - - processCalculatedFields(request, entityId); - processCalculatedFields(request, getProfileId(tenantId, entityId)); - - Map> tpiStatesToUpdate = new HashMap<>(); - processCalculatedFieldLinks(request, tpiStatesToUpdate); - if (!tpiStatesToUpdate.isEmpty()) { - tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(proto, ctxIds); - clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); - }); - } - } else { - clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(proto).build(), null); - } - } - } catch (Exception e) { - log.trace("Failed to update telemetry.", e); - } - } - private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { if (cfTargetEntityId != null) { calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId).forEach(ctx -> { @@ -483,7 +517,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); + TopicPartitionInfo targetEntityTpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, targetEntity); if (targetEntityTpi.isMyPartition()) { Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { @@ -495,31 +529,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - @Override - public void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto.getMsg()); - - if (proto.getLinksList().isEmpty()) { - onTelemetryUpdate(proto, callback); - return; - } - - proto.getLinksList().forEach(ctxIdProto -> { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } catch (Exception e) { - log.trace("Failed to process telemetry update msg: [{}]", proto, e); - } - } - private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); Map argumentValues = updatedTelemetry.entrySet().stream() @@ -534,7 +543,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); if (tpi.isMyPartition()) { log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { @@ -824,7 +833,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas List versions = result.getVersions(); for (int i = 0; i < entries.size(); i++) { long tsVersion = versions.get(i); - TsKvProto tsProto = ProtoUtils.toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); + TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); telemetryMsg.addTsData(tsProto); } msg.setTelemetryMsg(telemetryMsg.build()); @@ -858,8 +867,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - for (CalculatedFieldId cfId : calculatedFieldIds) { - telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + if (calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } } return telemetryMsg; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 4703ed1606..95b340c362 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -87,7 +88,7 @@ public class EntityStateSourcingListener { case ASSET -> { onAssetUpdate(event.getEntity(), event.getOldEntity()); } - case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE, CALCULATED_FIELD -> { + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -122,6 +123,9 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } + case CALCULATED_FIELD -> { + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity(), lifecycleEvent); + } default -> { } } @@ -146,7 +150,7 @@ public class EntityStateSourcingListener { Asset asset = (Asset) event.getEntity(); tbClusterService.onAssetDeleted(tenantId, asset, null); } - case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE, CALCULATED_FIELD -> { + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { @@ -187,6 +191,11 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } + case CALCULATED_FIELD -> { + CalculatedField calculatedField = (CalculatedField) event.getEntity(); + ComponentLifecycleMsg lifecycleMsg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); + tbClusterService.onCalculatedFieldDeleted(calculatedField.getTenantId(), calculatedField, lifecycleMsg); + } default -> { } } @@ -267,6 +276,15 @@ public class EntityStateSourcingListener { } } + private void onCalculatedFieldUpdate(Object entity, Object oldEntity, ComponentLifecycleEvent lifecycleEvent) { + CalculatedField calculatedField = (CalculatedField) entity; + CalculatedField oldCalculatedField = null; + if (oldEntity instanceof CalculatedField) { + oldCalculatedField = (CalculatedField) oldEntity; + } + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), lifecycleEvent)); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index e550a1154e..bd40784e8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,8 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -34,19 +32,16 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -170,6 +165,12 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer } else if (toCfMsg.hasLinkedTelemetryMsg()) { log.trace("[{}] Forwarding linked telemetry message for processing {}", id, toCfMsg.getLinkedTelemetryMsg()); forwardToCalculatedFieldService(toCfMsg.getLinkedTelemetryMsg(), callback); + } else if (toCfMsg.hasComponentLifecycleMsg()) { + log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); + forwardToCalculatedFieldService(toCfMsg.getComponentLifecycleMsg(), callback); + } else if (toCfMsg.hasEntityUpdateMsg()) { + log.trace("[{}] Forwarding entity update message for processing {}", id, toCfMsg.getEntityUpdateMsg()); + forwardToCalculatedFieldService(toCfMsg.getEntityUpdateMsg(), callback); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -219,36 +220,13 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); if (toCfNotification.hasComponentLifecycle()) { - forwardToCalculatedFieldService(toCfNotification.getComponentLifecycle(), callback); + handleComponentLifecycleMsg(id, ProtoUtils.fromProto(toCfNotification.getComponentLifecycle())); } else if (toCfNotification.hasEntityUpdateMsg()) { - forwardToCalculatedFieldService(toCfNotification.getEntityUpdateMsg(), callback); + processEntityUpdateMsg(toCfNotification.getEntityUpdateMsg()); } callback.onSuccess(); } - // private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { -// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); -// var oldProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getOldProfileIdMSB(), profileUpdateMsg.getOldProfileIdLSB())); -// var newProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getNewProfileIdMSB(), profileUpdateMsg.getNewProfileIdLSB())); -// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); -// } -// -// private void processProfileEntityMsg(TransportProtos.ProfileEntityMsgProto profileEntityMsg) { -// var tenantId = toTenantId(profileEntityMsg.getTenantIdMSB(), profileEntityMsg.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityType(), new UUID(profileEntityMsg.getEntityIdMSB(), profileEntityMsg.getEntityIdLSB())); -// var profileId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityProfileType(), new UUID(profileEntityMsg.getProfileIdMSB(), profileEntityMsg.getProfileIdLSB())); -// boolean added = profileEntityMsg.getAdded(); -// Set entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); -// if (added) { -// entitiesByProfile.add(entityId); -// } else { -// entitiesByProfile.remove(entityId); -// } -// } -// - private void forwardToCalculatedFieldService(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { var msg = linkedMsg.getMsg(); var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); @@ -275,7 +253,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } - private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { + private void forwardToCalculatedFieldService(ComponentLifecycleMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); @@ -287,7 +265,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } - private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { + private void forwardToCalculatedFieldService(CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); @@ -299,6 +277,23 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } + private void processEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto entityUpdateMsg) { + var tenantId = toTenantId(entityUpdateMsg.getTenantIdMSB(), entityUpdateMsg.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityType(), new UUID(entityUpdateMsg.getEntityIdMSB(), entityUpdateMsg.getEntityIdLSB())); + if (entityUpdateMsg.getAdded()) { + var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); + calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); + } else if (entityUpdateMsg.getDeleted()) { + var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); + calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); + } else if (entityUpdateMsg.getUpdated()) { + var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); + var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); + calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); + } + } + private void throwNotHandled(Object msg, TbCallback callback) { log.warn("Message not handled: {}", msg); callback.onFailure(new RuntimeException("Message not handled!")); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 04d6a53401..b1a4b1859e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,10 +38,12 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -68,6 +70,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -95,6 +98,7 @@ import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -145,6 +149,10 @@ public class DefaultTbClusterService implements TbClusterService { @Lazy private OtaPackageStateService otaPackageStateService; + @Autowired + @Lazy + private CalculatedFieldExecutionService calculatedFieldExecutionService; + private final TopicService topicService; private final TbDeviceProfileCache deviceProfileCache; private final TbAssetProfileCache assetProfileCache; @@ -342,21 +350,21 @@ public class DefaultTbClusterService implements TbClusterService { public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toCoreMsgs.incrementAndGet(); + toRuleEngineMsgs.incrementAndGet(); } @Override public void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { log.trace("PUSHING msg: {} to:{}", msg, tpi); producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); - toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + toRuleEngineMsgs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS } @Override public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toCoreMsgs.incrementAndGet(); + toRuleEngineNfs.incrementAndGet(); } @Override @@ -686,6 +694,20 @@ public class DefaultTbClusterService implements TbClusterService { broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } + @Override + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, ComponentLifecycleMsg lifecycleMsg) { + var created = oldCalculatedField == null; + calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); + broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + @Override + public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, ComponentLifecycleMsg lifecycleMsg) { + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); + broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { @@ -827,7 +849,7 @@ public class DefaultTbClusterService implements TbClusterService { } private void handleCalculatedFieldEntityUpdateEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId, boolean added, boolean updated, boolean deleted) { - TransportProtos.CalculatedFieldEntityUpdateMsgProto.Builder builder = TransportProtos.CalculatedFieldEntityUpdateMsgProto.newBuilder(); + CalculatedFieldEntityUpdateMsgProto.Builder builder = CalculatedFieldEntityUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setEntityType(entityId.getEntityType().name()); @@ -846,9 +868,22 @@ public class DefaultTbClusterService implements TbClusterService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - TransportProtos.CalculatedFieldEntityUpdateMsgProto msg = builder.build(); + CalculatedFieldEntityUpdateMsgProto msg = builder.build(); + + broadcastEntityUpdateEvent(msg); + pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); + } - pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); + private void broadcastEntityUpdateEvent(CalculatedFieldEntityUpdateMsgProto proto) { + TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); + Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); + for (String serviceId : tbCalculatedFieldServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(); + toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), toCfNotificationMsg), null); + toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 5b1e5d7d79..dac35bfc5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -178,16 +178,12 @@ public abstract class AbstractConsumerService> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; - private TbQueueProducer> toCalculatedFields; + private TbQueueProducer> toCalculatedFields; private TbQueueProducer> toCalculatedFieldNotifications; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { From d3278f05bb4658bc109b71b927afd2b78e2bea88 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Jan 2025 17:34:35 +0200 Subject: [PATCH 135/416] WIP: Cluster mode refactoring --- .../server/actors/ActorSystemContext.java | 6 + .../server/actors/app/AppActor.java | 25 ++ .../CalculatedFieldEntityActor.java | 65 ++++ .../CalculatedFieldEntityActorCreator.java | 49 +++ ...CalculatedFieldEntityMessageProcessor.java | 199 +++++++++++ .../CalculatedFieldLinkedTelemetryMsg.java | 40 +++ .../CalculatedFieldManagerActor.java | 84 +++++ .../CalculatedFieldManagerActorCreator.java | 46 +++ ...alculatedFieldManagerMessageProcessor.java | 149 +++++++++ .../CalculatedFieldStateRestoreMsg.java | 40 +++ .../CalculatedFieldTelemetryMsg.java | 39 +++ .../EntityCalculatedFieldTelemetryMsg.java | 55 +++ .../calculatedField/MultipleTbCallback.java | 49 +++ .../actors/service/DefaultActorService.java | 11 + .../server/actors/tenant/TenantActor.java | 29 +- .../cf/CalculatedFieldExecutionService.java | 23 +- .../cf/CalculatedFieldInitService.java | 19 ++ .../cf/DefaultCalculatedFieldCache.java | 2 + ...efaultCalculatedFieldExecutionService.java | 313 ++++++++---------- .../cf/DefaultCalculatedFieldInitService.java | 64 ++++ .../cf/ctx/CalculatedFieldEntityCtxId.java | 3 +- .../cf/ctx/CalculatedFieldStateService.java | 9 +- .../ctx/state/BaseCalculatedFieldState.java | 5 + .../cf/ctx/state/CalculatedFieldCtx.java | 1 + .../cf/ctx/state/CalculatedFieldState.java | 1 + .../cf/ctx/state/RocksDBStateService.java | 14 +- .../ctx/state/SingleValueArgumentEntry.java | 16 + ...faultTbCalculatedFieldConsumerService.java | 48 +-- .../DefaultTelemetrySubscriptionService.java | 4 +- .../src/main/resources/thingsboard.yml | 2 + .../resources/application-test.properties | 2 + .../TbCalculatedFieldEntityActorId.java | 56 ++++ .../cf/configuration/ReferencedEntityKey.java | 2 + .../server/common/msg/MsgType.java | 11 +- .../msg/ToCalculatedFieldSystemMsg.java | 27 ++ .../common/msg/cf/CalculatedFieldInitMsg.java | 34 ++ .../msg/cf/CalculatedFieldLinkInitMsg.java | 34 ++ .../server/queue/util/AfterStartUp.java | 4 + .../resources/application-test.properties | 2 + 39 files changed, 1362 insertions(+), 220 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java create mode 100644 common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index e6f541090c..09488bfe4e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -104,6 +104,7 @@ import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; @@ -507,6 +508,11 @@ public class ActorSystemContext { @Getter private EntityService entityService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 622812b327..9124dc0a9a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; @@ -111,6 +112,17 @@ public class AppActor extends ContextAwareActor { case SESSION_TIMEOUT_MSG: ctx.broadcastToChildrenByType(msg, EntityType.TENANT); break; + case CF_INIT_MSG: + case CF_LINK_INIT_MSG: + case CF_STATE_RESTORE_MSG: + case CF_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + break; + case CF_TELEMETRY_MSG: + case CF_LINKED_TELEMETRY_MSG: + case CF_ENTITY_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + break; default: return false; } @@ -175,6 +187,19 @@ public class AppActor extends ContextAwareActor { } } + private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { + if (priority) { + tenantActor.tellWithHighPriority(msg); + } else { + tenantActor.tell(msg); + } + }, () -> { + msg.getCallback().onSuccess(); + }); + } + + private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { if (priority) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java new file mode 100644 index 0000000000..43f057ad01 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorException; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; + +@Slf4j +public class CalculatedFieldEntityActor extends ContextAwareActor { + + private final CalculatedFieldEntityMessageProcessor processor; + + CalculatedFieldEntityActor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { + super(systemContext); + this.processor = new CalculatedFieldEntityMessageProcessor(systemContext, tenantId, entityId); + } + + @Override + public void init(TbActorCtx ctx) throws TbActorException { + super.init(ctx); + log.debug("[{}][{}] Starting CF entity actor.", processor.tenantId, processor.entityId); + try { + processor.init(ctx); + log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); + } catch (Exception e) { + log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); + throw new TbActorException("Failed to initialize device actor", e); + } + } + + @Override + protected boolean doProcess(TbActorMsg msg) { + switch (msg.getMsgType()) { + case CF_STATE_RESTORE_MSG: + processor.process((CalculatedFieldStateRestoreMsg) msg); + break; + case CF_ENTITY_TELEMETRY_MSG: + processor.process((EntityCalculatedFieldTelemetryMsg) msg); + break; + default: + return false; + } + return true; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java new file mode 100644 index 0000000000..c5f8ecf046 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActor; +import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbEntityActorId; +import org.thingsboard.server.actors.device.DeviceActor; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public class CalculatedFieldEntityActorCreator extends ContextBasedCreator { + + private final TenantId tenantId; + private final EntityId entityId; + + public CalculatedFieldEntityActorCreator(ActorSystemContext context, TenantId tenantId, EntityId entityId) { + super(context); + this.tenantId = tenantId; + this.entityId = entityId; + } + + @Override + public TbActorId createActorId() { + return new TbEntityActorId(entityId); + } + + @Override + public TbActor createActor() { + return new CalculatedFieldEntityActor(context, tenantId, entityId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java new file mode 100644 index 0000000000..6bb6256563 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -0,0 +1,199 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareMsgProcessor { + // (1 for result persistence + 1 for the state persistence ) + public static final int CALLBACKS_PER_CF = 2; + + final TenantId tenantId; + final EntityId entityId; + final CalculatedFieldExecutionService cfService; + + TbActorCtx ctx; + Map states = new HashMap<>(); + + CalculatedFieldEntityMessageProcessor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { + super(systemContext); + this.tenantId = tenantId; + this.entityId = entityId; + this.cfService = systemContext.getCalculatedFieldExecutionService(); + } + + void init(TbActorCtx ctx) { + this.ctx = ctx; + } + + public void process(EntityCalculatedFieldTelemetryMsg msg) { + var proto = msg.getProto(); + var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); + MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); + List cfIdList = getCalculatedFieldIds(proto); + Set cfIdSet = new HashSet<>(cfIdList); + for (var ctx : msg.getEntityIdFields()) { + process(ctx, proto, cfIdSet, cfIdList, callback); + } + for (var ctx : msg.getProfileIdFields()) { + process(ctx, proto, cfIdSet, cfIdList, callback); + } + } + + private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Set cfIds, List cfIdList, MultipleTbCallback callback) { + if (cfIds.contains(ctx.getCfId())) { + callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processTelemetry(ctx, proto, cfIdList, callback); + } else if (proto.getAttrDataCount() > 0) { + processAttributes(ctx, proto, cfIdList, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } + + @SneakyThrows + private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList())); + } + + @SneakyThrows + private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList())); + } + + private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, + Map newArgValues) throws InterruptedException, ExecutionException, TimeoutException { + if (newArgValues.isEmpty()) { + callback.onSuccess(CALLBACKS_PER_CF); + } + CalculatedFieldState state = getOrInitState(ctx); + if (state.updateState(newArgValues)) { + if (state.isReady()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + } else { + callback.onSuccess(); // State was updated but no calculation performed; + } + cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + + @SneakyThrows + private CalculatedFieldState getOrInitState(CalculatedFieldCtx ctx) { + CalculatedFieldState state = states.get(ctx.getCfId()); + if (state != null) { + return state; + } else { + ListenableFuture stateFuture = systemContext.getCalculatedFieldExecutionService().fetchStateFromDb(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + state = stateFuture.get(1, TimeUnit.MINUTES); + states.put(ctx.getCfId(), state); + } + return state; + } + + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { + Map arguments = new HashMap<>(); + var argNames = ctx.getMainEntityArguments(); + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); + argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + } + return arguments; + } + + private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { + Map arguments = new HashMap<>(); + var argNames = ctx.getMainEntityArguments(); + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + } + return arguments; + } + + private static List getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { + List cfIds = new LinkedList<>(); + for (var cfId : proto.getPreviousCalculatedFieldsList()) { + cfIds.add(new CalculatedFieldId(new UUID(cfId.getCalculatedFieldIdMSB(), cfId.getCalculatedFieldIdLSB()))); + } + return cfIds; + } + + public void process(CalculatedFieldStateRestoreMsg msg) { + states.put(msg.getId().cfId(), msg.getState()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java new file mode 100644 index 0000000000..0c352f5072 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; + +@Data +public class CalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldLinkedTelemetryMsgProto proto; + private final TbCallback callback; + + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINKED_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java new file mode 100644 index 0000000000..87292d3206 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorException; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; + +/** + * Created by ashvayka on 15.03.18. + */ +@Slf4j +public class CalculatedFieldManagerActor extends ContextAwareActor { + + private final CalculatedFieldManagerMessageProcessor processor; + + public CalculatedFieldManagerActor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.processor = new CalculatedFieldManagerMessageProcessor(systemContext, tenantId); + } + + @Override + public void init(TbActorCtx ctx) throws TbActorException { + super.init(ctx); + log.debug("[{}] Starting CF manager actor.", processor.tenantId); + try { + processor.init(ctx); + log.debug("[{}] CF manager actor started.", processor.tenantId); + } catch (Exception e) { + log.warn("[{}] Unknown failure", processor.tenantId, e); + throw new TbActorException("Failed to initialize manager actor", e); + } + } + + @Override + protected boolean doProcess(TbActorMsg msg) { + switch (msg.getMsgType()) { + case PARTITION_CHANGE_MSG: + ctx.broadcastToChildren(msg, true); // TODO + break; + case CF_INIT_MSG: + processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); + break; + case CF_LINK_INIT_MSG: + processor.onLinkInitMsg((CalculatedFieldLinkInitMsg) msg); + break; + case CF_STATE_RESTORE_MSG: + processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); + break; + case CF_UPDATE_MSG: +// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); + break; + case CF_TELEMETRY_MSG: + processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); + break; + case CF_LINKED_TELEMETRY_MSG: + case CF_ENTITY_UPDATE_MSG: +// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); + break; + default: + return false; + } + return true; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java new file mode 100644 index 0000000000..2a53089bed --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActor; +import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbEntityActorId; +import org.thingsboard.server.actors.TbStringActorId; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public class CalculatedFieldManagerActorCreator extends ContextBasedCreator { + + private final TenantId tenantId; + + public CalculatedFieldManagerActorCreator(ActorSystemContext context, TenantId tenantId) { + super(context); + this.tenantId = tenantId; + } + + @Override + public TbActorId createActorId() { + return new TbStringActorId("CFM|" + tenantId); + } + + @Override + public TbActor createActor() { + return new CalculatedFieldManagerActor(context, tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java new file mode 100644 index 0000000000..4bc0589e95 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -0,0 +1,149 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; + + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { + + private final Map calculatedFields = new HashMap<>(); + private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + + protected TbActorCtx ctx; + final TenantId tenantId; + + CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.assetProfileCache = systemContext.getAssetProfileCache(); + this.deviceProfileCache = systemContext.getDeviceProfileCache(); + this.tenantId = tenantId; + } + + void init(TbActorCtx ctx) { + this.ctx = ctx; + } + + public void onFieldInitMsg(CalculatedFieldInitMsg msg) { + var cf = msg.getCf(); + calculatedFields.put(cf.getId(), cf); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()) + .add(new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService())); + msg.getCallback().onSuccess(); + } + + public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { + var link = msg.getLink(); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); + msg.getCallback().onSuccess(); + } + + public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) { + if (calculatedFields.containsKey(msg.getId().cfId())) { + getOrCreateActor(msg.getId().entityId()).tell(msg); + } else { + // TODO: remove state from storage + } + } + + public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + var proto = msg.getProto(); + // process all cfs related to entity, or it's profile; + var entityIdFields = getCalculatedFieldsByEntityId(entityId); + var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + //TODO: Transfer only 'part' of the original callback. + getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, msg.getCallback())); + // process all links (if any); + var links = getCalculatedFieldLinksByEntityId(entityId); + } + + private List getCalculatedFieldsByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + var result = entityIdCalculatedFields.get(entityId); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + private List getCalculatedFieldLinksByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + var result = entityIdCalculatedFieldLinks.get(entityId); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + + protected TbActorRef getOrCreateActor(EntityId entityId) { + return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), + () -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, + () -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), + () -> true); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java new file mode 100644 index 0000000000..7c0e67f0ec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +@Data +public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMsg { + + private final CalculatedFieldEntityCtxId id; + private final CalculatedFieldState state; + + @Override + public MsgType getMsgType() { + return MsgType.CF_STATE_RESTORE_MSG; + } + + @Override + public TenantId getTenantId() { + return id.tenantId(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java new file mode 100644 index 0000000000..5cce83b00b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; + +@Data +public class CalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final TbCallback callback; + + + @Override + public MsgType getMsgType() { + return MsgType.CF_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java new file mode 100644 index 0000000000..8465bd8fcc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityCalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final List entityIdFields; + private final List profileIdFields; + private final TbCallback callback; + + public EntityCalculatedFieldTelemetryMsg(CalculatedFieldTelemetryMsg msg, + List entityIdFields, + List profileIdFields, + TbCallback callback) { + this.tenantId = msg.getTenantId(); + this.entityId = msg.getEntityId(); + this.proto = msg.getProto(); + this.entityIdFields = entityIdFields; + this.profileIdFields = profileIdFields; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java new file mode 100644 index 0000000000..18e700f38c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.common.msg.queue.TbCallback; + +import java.util.concurrent.atomic.AtomicInteger; + +public class MultipleTbCallback implements TbCallback { + + private final AtomicInteger counter; + private final TbCallback callback; + + public MultipleTbCallback(int count, TbCallback callback) { + this.counter = new AtomicInteger(count); + this.callback = callback; + } + + @Override + public void onSuccess() { + if (counter.decrementAndGet() <= 0) { + callback.onSuccess(); + } + } + + public void onSuccess(int number) { + if (counter.addAndGet(-number) <= 0) { + callback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 492940a274..7e0d1b4447 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -49,6 +49,8 @@ public class DefaultActorService extends TbApplicationEventListener deletedDevices; + private TbActorRef cfActor; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext, tenantId); @@ -95,6 +98,11 @@ public class TenantActor extends RuleChainManagerActor { } else { log.info("[{}] Skip init of the rule chains due to API limits", tenantId); } + //TODO: IM - extend API usage to have CF Exec Enabled? Not in 4.0; + cfActor = ctx.getOrCreateChildActor(new TbStringActorId("CFM|" + tenantId), + () -> DefaultActorService.CF_MANAGER_DISPATCHER_NAME, + () -> new CalculatedFieldManagerActorCreator(systemContext, tenantId), + () -> true); } catch (Exception e) { log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); cantFindTenant = true; @@ -159,12 +167,31 @@ public class TenantActor extends RuleChainManagerActor { case RULE_CHAIN_TO_RULE_CHAIN_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; + case CF_INIT_MSG: + case CF_LINK_INIT_MSG: + case CF_STATE_RESTORE_MSG: + case CF_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + break; + case CF_TELEMETRY_MSG: + case CF_LINKED_TELEMETRY_MSG: + case CF_ENTITY_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + break; default: return false; } return true; } + private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + if (priority) { + cfActor.tellWithHighPriority(msg); + } else { + cfActor.tell(msg); + } + } + private boolean isMyPartition(EntityId entityId) { return systemContext.resolve(ServiceType.TB_CORE, tenantId, entityId).isMyPartition(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 62e234943b..d38c1c58ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,14 +15,22 @@ */ package org.thingsboard.server.service.cf; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.List; @@ -30,16 +38,17 @@ public interface CalculatedFieldExecutionService { /** * Filter CFs based on the request entity. Push to the queue if any matching CF exist; - * @param request - telemetry save request; - * @param request - telemetry save result; + * + * @param request - telemetry save request; + * @param callback */ - void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); - void pushRequestToQueue(AttributesSaveRequest request, List result); + void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); - void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback); + void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback); + ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); @@ -53,4 +62,6 @@ public interface CalculatedFieldExecutionService { void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java new file mode 100644 index 0000000000..25f92e797f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +public interface CalculatedFieldInitService { +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 4278e74845..b0c9f6adde 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; @@ -36,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f92b9819ed..bb8ba8bb63 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -42,7 +41,6 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -63,7 +61,6 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; -import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; @@ -108,12 +105,13 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; @@ -167,7 +165,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(() -> states.putAll(stateService.restoreStates())); } @PreDestroy @@ -192,22 +189,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @Override - public void pushRequestToQueue(AttributesSaveRequest request, List result) { + public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, @@ -241,77 +238,84 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { - - callback.onSuccess(); + public ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { + Map> argFutures = new HashMap<>(); + for (var entry : ctx.getArguments().entrySet()) { + var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; + var argValueFuture = fetchKvEntry(ctx.getTenantId(), argEntityId, entry.getValue()); + argFutures.put(entry.getKey(), argValueFuture); + } + return Futures.whenAllComplete(argFutures.values()).call(() -> { + var result = createStateByType(ctx.getCfType()); + result.updateState(argFutures.entrySet().stream() + .collect(Collectors.toMap( + Entry::getKey, // Keep the key as is + entry -> { + try { + // Resolve the future to get the value + return entry.getValue().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); + } + } + ))); + return result; + }, calculatedFieldCallbackExecutor); } @Override - public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { - - callback.onSuccess(); + public void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + stateService.persistState(stateId, state, callback); } @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - Map> tpiTargetEntityMap = new HashMap<>(); - - for (CalculatedField cf : cfs) { - - Consumer resolvePartition = entityId -> { - TopicPartitionInfo tpi; - try { - tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); - if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { - tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); - } - } catch (Exception e) { - log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", - entityId, cf.getTenantId(), e.getMessage()); - } - }; - - EntityId cfEntityId = cf.getEntityId(); - if (isProfileEntity(cfEntityId)) { - calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); - } else { - resolvePartition.accept(cfEntityId); - } - } - - for (var entry : tpiTargetEntityMap.entrySet()) { - for (List partition : Lists.partition(entry.getValue(), 1000)) { - log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); - var future = calculatedFieldExecutor.submit(() -> { - try { - for (CalculatedFieldEntityCtxId ctxId : partition) { - restoreState(ctxId.cfId(), ctxId.entityId()); - } - } catch (Throwable t) { - log.error("Unexpected exception while restoring CalculatedField states", t); - throw t; - } - }); - result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); - } - } +// PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); +// Map> tpiTargetEntityMap = new HashMap<>(); +// +// for (CalculatedField cf : cfs) { +// +// Consumer resolvePartition = entityId -> { +// TopicPartitionInfo tpi; +// try { +// tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); +// if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { +// tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); +// } +// } catch (Exception e) { +// log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", +// entityId, cf.getTenantId(), e.getMessage()); +// } +// }; +// +// EntityId cfEntityId = cf.getEntityId(); +// if (isProfileEntity(cfEntityId)) { +// calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); +// } else { +// resolvePartition.accept(cfEntityId); +// } +// } +// +// for (var entry : tpiTargetEntityMap.entrySet()) { +// for (List partition : Lists.partition(entry.getValue(), 1000)) { +// log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); +// var future = calculatedFieldExecutor.submit(() -> { +// try { +// for (CalculatedFieldEntityCtxId ctxId : partition) { +// restoreState(ctxId.cfId(), ctxId.entityId()); +// } +// } catch (Throwable t) { +// log.error("Unexpected exception while restoring CalculatedField states", t); +// throw t; +// } +// }); +// result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); +// } +// } return result; } - private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - CalculatedFieldEntityCtx restoredCtx = stateService.restoreState(ctxId); - - if (restoredCtx != null) { - states.put(ctxId, restoredCtx); - log.info("Restored state for CalculatedField [{}]", calculatedFieldId); - } else { - log.warn("No state found for CalculatedField [{}], entity [{}].", calculatedFieldId, entityId); - } - } - @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { cleanupEntity(entityId); @@ -491,7 +495,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getCfId(), targetEntity)); + ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getTenantId(), ctx.getCfId(), targetEntity)); } } @@ -525,7 +529,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); +// updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); } @Override @@ -569,9 +573,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - states.remove(ctxId); - stateService.removeState(ctxId); } private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { @@ -601,7 +602,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); +// updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); callback.onSuccess(); } @@ -613,96 +614,83 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { - CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); - Map argumentsMap = new HashMap<>(argumentValues); - - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); - - states.compute(entityCtxId, (ctxId, ctx) -> { - CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); - - CompletableFuture updateFuture = new CompletableFuture<>(); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentsMap)) { - calculatedFieldEntityCtx.setState(state); - stateService.persistState(entityCtxId, calculatedFieldEntityCtx); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); - } - log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); - } - updateFuture.complete(null); - }; - - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - - boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); - - if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); - } - - try { - updateFuture.join(); - } catch (Exception e) { - log.trace("Failed to update state for ctxId [{}].", ctxId, e); - throw new RuntimeException("Failed to update or initialize state.", e); - } - - return calculatedFieldEntityCtx; - }); - } - - private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List previousCalculatedFieldIds) { - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, previousCalculatedFieldIds); - } - } +// private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { +// CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); +// Map argumentsMap = new HashMap<>(argumentValues); +// +// CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); +// +// states.compute(entityCtxId, (ctxId, ctx) -> { +// CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); +// +// CompletableFuture updateFuture = new CompletableFuture<>(); +// +// Consumer performUpdateState = (state) -> { +// if (state.updateState(argumentsMap)) { +// calculatedFieldEntityCtx.setState(state); +// stateService.persistState(entityCtxId, calculatedFieldEntityCtx); +// Map arguments = state.getArguments(); +// boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && +// !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); +// if (allArgsPresent) { +// performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); +// } +// log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); +// } +// updateFuture.complete(null); +// }; +// +// CalculatedFieldState state = calculatedFieldEntityCtx.getState(); +// +// boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); +// boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() +// .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); +// +// if (!allKeysPresent || requiresTsRollingUpdate) { +// Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() +// .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) +// .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); +// +// fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) +// .addListener(() -> performUpdateState.accept(state), +// calculatedFieldCallbackExecutor); +// } else { +// performUpdateState.accept(state); +// } +// +// try { +// updateFuture.join(); +// } catch (Exception e) { +// log.trace("Failed to update state for ctxId [{}].", ctxId, e); +// throw new RuntimeException("Failed to update or initialize state.", e); +// } +// +// return calculatedFieldEntityCtx; +// }); +// } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); - } - }, MoreExecutors.directExecutor()); - } - - private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List previousCalculatedFieldIds) { + @Override + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { try { OutputType type = calculatedFieldResult.getType(); TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); - if (previousCalculatedFieldIds != null && previousCalculatedFieldIds.contains(calculatedFieldId)) { - throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); - } - List calculatedFieldIds = previousCalculatedFieldIds != null - ? new ArrayList<>(previousCalculatedFieldIds) - : new ArrayList<>(); - calculatedFieldIds.add(calculatedFieldId); - TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).previousCalculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); - clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); - log.info("Pushed message to rule engine: originatorId=[{}]", originatorId); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); + clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(); + log.trace("[{}][{}] Pushed message to rule engine: {} ", tenantId, entityId, msg); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }); } catch (Exception e) { - log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); + log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); } } @@ -781,15 +769,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - CalculatedFieldEntityCtx state = stateService.restoreState(entityCtxId); - - if (state == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); - } - return state; - } - private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java new file mode 100644 index 0000000000..87dda7013f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; + +@Service +@TbRuleEngineComponent +@RequiredArgsConstructor +public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { + + private final CalculatedFieldService calculatedFieldService; + private final CalculatedFieldStateService stateService; + private final ActorSystemContext actorSystemContext; + + @Value("${calculated_fields.init_fetch_pack_size:50000}") + @Getter + private int initFetchPackSize; + + @AfterStartUp(order = AfterStartUp.CF_INIT_SERVICE) + public void initCalculatedFieldDefinitions() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf))); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link))); + //TODO: combine with the DefaultCalculatedFieldCache. + + } + + @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) + public void initCalculatedFieldStates() { + stateService.restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); + } + + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index 5fb90a3e46..6ce0a11ade 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; -public record CalculatedFieldEntityCtxId(CalculatedFieldId cfId, EntityId entityId) { +public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index 8bc5756f4e..2bf6b7e0f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -15,15 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + import java.util.Map; public interface CalculatedFieldStateService { - Map restoreStates(); + Map restoreStates(); - CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId); + CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId); - void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state); + void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId ctxId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index bc9a421e47..ca243e25b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -66,4 +66,9 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { return stateUpdated; } + @Override + public boolean isReady() { + //TODO: IM + return true; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e4abbee4cd..1a0a16254a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import java.util.ArrayList; import java.util.HashMap; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 3c4a680df1..4b7918cc03 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -44,4 +44,5 @@ public interface CalculatedFieldState { ListenableFuture performCalculation(CalculatedFieldCtx ctx); + boolean isReady(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index db8950804c..aa82f2fc57 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -36,24 +37,25 @@ public class RocksDBStateService implements CalculatedFieldStateService { private final RocksDBService rocksDBService; @Override - public Map restoreStates() { + public Map restoreStates() { return rocksDBService.getAll().entrySet().stream() .collect(Collectors.toMap( entry -> JacksonUtil.fromString(entry.getKey(), CalculatedFieldEntityCtxId.class), - entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldEntityCtx.class) + entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldState.class) )); } @Override - public CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId) { + public CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId) { return Optional.ofNullable(rocksDBService.get(JacksonUtil.writeValueAsString(ctxId))) - .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class)) + .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldState.class)) .orElse(null); } @Override - public void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state) { - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(state)); + public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback){ + rocksDBService.put(JacksonUtil.writeValueAsString(stateId), JacksonUtil.writeValueAsString(state)); + callback.onSuccess(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 8d5080e90f..d5f7b1aee6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -21,6 +21,10 @@ import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @Data @NoArgsConstructor @@ -34,6 +38,18 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private Long version; + public SingleValueArgumentEntry(TsKvProto entry) { + this.ts = entry.getTs(); + this.version = entry.getVersion(); + this.value = ProtoUtils.fromProto(entry).getValue(); + } + + public SingleValueArgumentEntry(AttributeValueProto entry) { + this.ts = entry.getLastUpdateTs(); + this.version = entry.getVersion(); + this.value = ProtoUtils.fromProto(entry).getValue(); + } + public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry tsKvEntry) { this.ts = tsKvEntry.getTs(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index e550a1154e..fb1d6b19ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,8 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -30,23 +28,20 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -166,10 +161,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer pendingMsgHolder.setMsg(toCfMsg); if (toCfMsg.hasTelemetryMsg()) { log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); - forwardToCalculatedFieldService(toCfMsg.getTelemetryMsg(), callback); + forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); } else if (toCfMsg.hasLinkedTelemetryMsg()) { log.trace("[{}] Forwarding linked telemetry message for processing {}", id, toCfMsg.getLinkedTelemetryMsg()); - forwardToCalculatedFieldService(toCfMsg.getLinkedTelemetryMsg(), callback); + forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -219,9 +214,9 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); if (toCfNotification.hasComponentLifecycle()) { - forwardToCalculatedFieldService(toCfNotification.getComponentLifecycle(), callback); + forwardToActorSystem(toCfNotification.getComponentLifecycle(), callback); } else if (toCfNotification.hasEntityUpdateMsg()) { - forwardToCalculatedFieldService(toCfNotification.getEntityUpdateMsg(), callback); + forwardToActorSystem(toCfNotification.getEntityUpdateMsg(), callback); } callback.onSuccess(); } @@ -249,33 +244,20 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer // } // - private void forwardToCalculatedFieldService(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { - var msg = linkedMsg.getMsg(); + private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onLinkedTelemetryMsg(linkedMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + actorContext.tell(new CalculatedFieldTelemetryMsg(tenantId, entityId, msg, callback)); } - private void forwardToCalculatedFieldService(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + private void forwardToActorSystem(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + var msg = linkedMsg.getMsg(); var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onTelemetryMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); } - private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { + private void forwardToActorSystem(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); @@ -287,7 +269,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } - private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { + private void forwardToActorSystem(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index df18640359..3f7f9c0b7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -147,7 +147,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } DonAsynchron.withCallback(resultFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result); + calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { @@ -167,7 +167,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); DonAsynchron.withCallback(saveFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result); + calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index d62aac5e6c..30a3e51d8c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -441,6 +441,8 @@ actors: device_dispatcher_pool_size: "${ACTORS_SYSTEM_DEVICE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for device actors rule_dispatcher_pool_size: "${ACTORS_SYSTEM_RULE_DISPATCHER_POOL_SIZE:8}" # Thread pool size for actor system dispatcher that process messages for rule engine (chain/node) actors edge_dispatcher_pool_size: "${ACTORS_SYSTEM_EDGE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for edge actors + cfm_dispatcher_pool_size: "${ACTORS_SYSTEM_CFM_DISPATCHER_POOL_SIZE:2}" # Thread pool size for actor system dispatcher that process messages for CalculatedField manager actors + cfe_dispatcher_pool_size: "${ACTORS_SYSTEM_CFE_DISPATCHER_POOL_SIZE:8}" # Thread pool size for actor system dispatcher that process messages for CalculatedField entity actors tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" # Create components in initialization session: diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 9951caa876..a1315cc8bd 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -35,6 +35,8 @@ sql.events.batch_threads=2 actors.system.tenant_dispatcher_pool_size=4 actors.system.device_dispatcher_pool_size=8 actors.system.rule_dispatcher_pool_size=12 +actors.system.cfm_dispatcher_pool_size=2 +actors.system.cfe_dispatcher_pool_size=2 transport.sessions.report_timeout=10000 queue.transport_api.request_poll_interval=5 queue.transport_api.response_poll_interval=5 diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java new file mode 100644 index 0000000000..66522cd08d --- /dev/null +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Objects; + +public class TbCalculatedFieldEntityActorId implements TbActorId { + + @Getter + private final EntityId entityId; + + public TbCalculatedFieldEntityActorId(EntityId entityId) { + this.entityId = entityId; + } + + @Override + public String toString() { + return entityId.getEntityType() + "|" + entityId.getId(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TbCalculatedFieldEntityActorId that = (TbCalculatedFieldEntityActorId) o; + return entityId.equals(that.entityId); + } + + @Override + public int hashCode() { + // Magic number to ensure that the hash does not match with the hash of other actor id - (TbEntityActorId) + return 42 + Objects.hash(entityId); + } + + @Override + public EntityType getEntityType() { + return entityId.getEntityType(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java index b49495d959..b4bcc77a17 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -27,4 +27,6 @@ public class ReferencedEntityKey { private ArgumentType type; private AttributeScope scope; + + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index b05bb75032..91419bee9c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -133,7 +133,16 @@ public enum MsgType { * Messages that are sent to and from edge session to start edge synchronization process */ EDGE_SYNC_REQUEST_TO_EDGE_SESSION_MSG, - EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG; + EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG, + + CF_INIT_MSG, // Sent to init particular calculated field; + CF_LINK_INIT_MSG, // Sent to init particular calculated field; + CF_STATE_RESTORE_MSG,// Sent to init particular calculated field entity state; + CF_TELEMETRY_MSG, + CF_ENTITY_TELEMETRY_MSG, + CF_LINKED_TELEMETRY_MSG, + CF_UPDATE_MSG, + CF_ENTITY_UPDATE_MSG; @Getter private final boolean ignoreOnStart; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java new file mode 100644 index 0000000000..2da959ec7d --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg; + +import org.thingsboard.server.common.msg.aware.TenantAwareMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +public interface ToCalculatedFieldSystemMsg extends TenantAwareMsg { + + default TbCallback getCallback() { + return TbCallback.EMPTY; + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java new file mode 100644 index 0000000000..4d9f4ce414 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedField cf; + + @Override + public MsgType getMsgType() { + return MsgType.CF_INIT_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java new file mode 100644 index 0000000000..7f582e6ff8 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldLinkInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldLink link; + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINK_INIT_MSG; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index 53ff839c49..70ae853e1e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -38,6 +38,10 @@ public @interface AfterStartUp { int ACTOR_SYSTEM = 9; int REGULAR_SERVICE = 10; + int CF_INIT_SERVICE = 10; + int CF_STATE_RESTORE_SERVICE = 11; + int CF_CONSUMER_SERVICE = 12; + int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; int AFTER_TRANSPORT_SERVICE = Integer.MAX_VALUE - 999; diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 1e11c77f9a..a1bb335ad0 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -145,6 +145,8 @@ sql.events.batch_threads=2 actors.system.tenant_dispatcher_pool_size=4 actors.system.device_dispatcher_pool_size=8 actors.system.rule_dispatcher_pool_size=12 +actors.system.cfm_dispatcher_pool_size=2 +actors.system.cfe_dispatcher_pool_size=2 transport.sessions.report_timeout=10000 queue.transport_api.request_poll_interval=5 queue.transport_api.response_poll_interval=5 From 3d42a4ca04af2c21a89fc7fa11a8f9fd7adb970d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Jan 2025 17:57:39 +0200 Subject: [PATCH 136/416] Actor system implementation draft --- .../CalculatedFieldEntityMessageProcessor.java | 8 ++++---- .../cf/DefaultCalculatedFieldExecutionService.java | 6 ++++-- .../service/cf/ctx/state/CalculatedFieldState.java | 1 + .../org/thingsboard/server/common/util/ProtoUtils.java | 10 ++++++---- .../sql/cf/DefaultNativeCalculatedFieldRepository.java | 2 +- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 6bb6256563..533eeb5e31 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -78,6 +78,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM this.ctx = ctx; } + public void process(CalculatedFieldStateRestoreMsg msg) { + states.put(msg.getId().cfId(), msg.getState()); + } + public void process(EntityCalculatedFieldTelemetryMsg msg) { var proto = msg.getProto(); var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); @@ -192,8 +196,4 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } return cfIds; } - - public void process(CalculatedFieldStateRestoreMsg msg) { - states.put(msg.getId().cfId(), msg.getState()); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index bb8ba8bb63..3e37366da3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -837,8 +837,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - for (CalculatedFieldId cfId : calculatedFieldIds) { - telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + if(calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } } return telemetryMsg; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 4b7918cc03..f261e586a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -44,5 +44,6 @@ public interface CalculatedFieldState { ListenableFuture performCalculation(CalculatedFieldCtx ctx); + @JsonIgnore boolean isReady(); } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 6a40f13688..9bbce7c522 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -644,11 +644,13 @@ public class ProtoUtils { } public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { - return TransportProtos.TsKvProto.newBuilder() + var builder = TransportProtos.TsKvProto.newBuilder() .setTs(tsKvEntry.getTs()) - .setKv(toKeyValueProto(tsKvEntry)) - .setVersion(tsKvEntry.getVersion()) - .build(); + .setKv(toKeyValueProto(tsKvEntry)); + if (tsKvEntry.getVersion() != null) { + builder.setVersion(tsKvEntry.getVersion()); + } + return builder.build(); } public static TransportProtos.KeyValueProto toKeyValueProto(KvEntry kvEntry) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index bb88982e3d..677234dc20 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -77,7 +77,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF String name = (String) row.get("name"); int configurationVersion = (int) row.get("configuration_version"); JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); - long version = (long) row.get("version"); + long version = row.get("version") != null ? (long) row.get("version") : 0; Object externalIdObj = row.get("external_id"); CalculatedField calculatedField = new CalculatedField(); From d54cb300d42757b929bbe94936e6d90b1f5db874 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 28 Jan 2025 08:43:29 +0200 Subject: [PATCH 137/416] added new endpoint --- .../controller/CalculatedFieldController.java | 30 +++++++++++++++++++ .../controller/ControllerConstants.java | 1 + 2 files changed, 31 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 4bf243034b..efe25e3883 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; @@ -24,17 +25,26 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; +import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -81,6 +91,26 @@ public class CalculatedFieldController extends BaseController { return calculatedField; } + @ApiOperation(value = "Get Calculated Fields (getCalculatedFields)", + notes = "Returns a page of calculated fields. " + PAGE_DATA_PARAMETERS + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) + @ResponseBody + public PageData getCalculatedFields( + @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) + @RequestParam(required = false) String sortProperty, + @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + return checkNotNull(calculatedFieldService.findAllCalculatedFields(pageLink)); + } @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index c7a59cfd83..29625941bd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -96,6 +96,7 @@ public class ControllerConstants { protected static final String EDGE_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the edge name."; protected static final String EVENT_TEXT_SEARCH_DESCRIPTION = "The value is not used in searching."; protected static final String AUDIT_LOG_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on one of the next properties: entityType, entityName, userName, actionType, actionStatus."; + protected static final String CF_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the calculated field name."; protected static final String SORT_PROPERTY_DESCRIPTION = "Property of entity to sort by"; protected static final String SORT_ORDER_DESCRIPTION = "Sort order. ASC (ASCENDING) or DESC (DESCENDING)"; From 5e16db275cb628bf57d3afea6c2df78bcba86fd3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 29 Jan 2025 14:44:28 +0200 Subject: [PATCH 138/416] state and argument refactoring --- ...efaultCalculatedFieldExecutionService.java | 11 +- .../service/cf/ctx/state/ArgumentEntry.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 38 +-- .../cf/ctx/state/CalculatedFieldCtx.java | 5 +- .../cf/ctx/state/CalculatedFieldState.java | 1 - .../ctx/state/ScriptCalculatedFieldState.java | 9 +- .../ctx/state/SimpleCalculatedFieldState.java | 26 +- .../ctx/state/SingleValueArgumentEntry.java | 26 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 48 ++-- .../state/ScriptCalculatedFieldStateTest.java | 238 ++++++++++++++++++ .../state/SimpleCalculatedFieldStateTest.java | 213 ++++++++++++++++ .../state/SingleValueArgumentEntryTest.java | 71 ++++++ .../ctx/state/TsRollingArgumentEntryTest.java | 93 +++++++ 13 files changed, 725 insertions(+), 56 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f333eecbd8..6f19090b6e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; @@ -248,7 +247,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas argFutures.put(entry.getKey(), argValueFuture); } return Futures.whenAllComplete(argFutures.values()).call(() -> { - var result = createStateByType(ctx.getCfType()); + var result = createStateByType(ctx); result.updateState(argFutures.entrySet().stream() .collect(Collectors.toMap( Entry::getKey, // Keep the key as is @@ -798,10 +797,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return payload; } - private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { - return switch (calculatedFieldType) { - case SIMPLE -> new SimpleCalculatedFieldState(); - case SCRIPT -> new ScriptCalculatedFieldState(); + private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { + return switch (ctx.getCfType()) { + case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); + case SCRIPT -> new ScriptCalculatedFieldState(ctx.getArgNames()); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index b261840bfd..be471a8ad9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -39,7 +39,7 @@ public interface ArgumentEntry { Object getValue(); - boolean hasUpdatedValue(ArgumentEntry entry); + boolean updateEntry(ArgumentEntry entry); static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { return new SingleValueArgumentEntry(kvEntry); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index ca243e25b5..d623043cee 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -16,14 +16,21 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; +import java.util.List; import java.util.Map; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { + protected List requiredArguments; protected Map arguments; public BaseCalculatedFieldState() { - arguments = new HashMap<>(); + this.arguments = new HashMap<>(); + } + + public BaseCalculatedFieldState(List requiredArguments) { + this.requiredArguments = requiredArguments; + this.arguments = new HashMap<>(); } @Override @@ -44,22 +51,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { ArgumentEntry newEntry = entry.getValue(); ArgumentEntry existingEntry = arguments.get(key); - if (existingEntry == null || existingEntry.hasUpdatedValue(newEntry)) { - if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof TsRollingArgumentEntry newTsRollingEntry) { - existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); - } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { - existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); - } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { -// Long existingVersion = existingSingleValueEntry.getVersion(); -// Long newVersion = singleValueEntry.getVersion(); -// if (newVersion != null && (existingVersion == null || newVersion > existingVersion)) { -// arguments.put(key, newEntry.copy()); -// } - arguments.put(key, newEntry.copy()); - } else { - arguments.put(key, newEntry.copy()); - } + if (existingEntry == null) { + validateNewEntry(newEntry); + arguments.put(key, newEntry.copy()); stateUpdated = true; + } else { + stateUpdated = existingEntry.updateEntry(newEntry); } } @@ -68,7 +65,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @Override public boolean isReady() { - //TODO: IM - return true; + return arguments.keySet().containsAll(requiredArguments) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && + !arguments.containsValue(TsRollingArgumentEntry.EMPTY); } + + protected void validateNewEntry(ArgumentEntry newEntry) { + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 1a0a16254a..b9834f18d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; +import net.objecthunter.exp4j.Expression; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; @@ -31,7 +32,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import java.util.ArrayList; import java.util.HashMap; @@ -56,6 +56,7 @@ public class CalculatedFieldCtx { private String expression; private TbelInvokeService tbelInvokeService; private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + private ThreadLocal customExpression; public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { this.cfId = calculatedField.getId(); @@ -86,6 +87,8 @@ public class CalculatedFieldCtx { this.tbelInvokeService = tbelInvokeService; if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + } else { + this.customExpression = new ThreadLocal<>(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index f261e586a4..4b7918cc03 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -44,6 +44,5 @@ public interface CalculatedFieldState { ListenableFuture performCalculation(CalculatedFieldCtx ctx); - @JsonIgnore boolean isReady(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 0421055fef..290fd95370 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -32,6 +33,10 @@ import java.util.TreeMap; @Slf4j public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { + public ScriptCalculatedFieldState(List requiredArguments) { + super(requiredArguments); + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -40,9 +45,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { arguments.forEach((key, argumentEntry) -> { - if (argumentEntry instanceof TsRollingArgumentEntry) { + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingEntry) { Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + TreeMap tsRecords = tsRollingEntry.getTsRecords(); if (tsRecords.size() > argument.getLimit()) { tsRecords.pollFirstEntry(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index e16d310b3e..59b3009bb4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -24,21 +24,32 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; +import java.util.List; import java.util.Map; @Data public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { + public SimpleCalculatedFieldState(List requiredArguments) { + super(requiredArguments); + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + if (newEntry instanceof TsRollingArgumentEntry) { + throw new IllegalArgumentException("Rolling argument entry is not supported for simple calculated fields."); + } + } + @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { String expression = ctx.getExpression(); - ThreadLocal customExpression = new ThreadLocal<>(); + ThreadLocal customExpression = ctx.getCustomExpression(); var expr = customExpression.get(); if (expr == null) { expr = new ExpressionBuilder(expression) @@ -47,9 +58,14 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { .build(); customExpression.set(expr); } - Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); - expr.setVariables(variables); + + for (Map.Entry entry : this.arguments.entrySet()) { + try { + expr.setVariable(entry.getKey(), Double.parseDouble(entry.getValue().getValue().toString())); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); + } + } double expressionResult = expr.evaluate(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index d5f7b1aee6..5117fcab0e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -21,7 +21,6 @@ import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @@ -35,7 +34,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - private Long version; public SingleValueArgumentEntry(TsKvProto entry) { @@ -79,14 +77,28 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return value; } - @Override - public boolean hasUpdatedValue(ArgumentEntry entry) { - return this.ts != ((SingleValueArgumentEntry) entry).getTs(); - } - @Override public ArgumentEntry copy() { return new SingleValueArgumentEntry(this.ts, this.value, this.version); } + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof SingleValueArgumentEntry singleValueEntry) { + if (singleValueEntry.getTs() == this.ts) { + return false; + } + + Long newVersion = singleValueEntry.getVersion(); + if (newVersion == null || this.version == null || newVersion > this.version) { + this.ts = singleValueEntry.getTs(); + this.value = singleValueEntry.getValue(); + this.version = newVersion; + return true; + } + } else { + throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + } + return false; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1118e3af13..b86a51ca03 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -62,31 +62,49 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public boolean hasUpdatedValue(ArgumentEntry entry) { - return entry instanceof SingleValueArgumentEntry ? - !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()) : - !tsRecords.keySet().containsAll(((TsRollingArgumentEntry) entry).getTsRecords().keySet()); + public ArgumentEntry copy() { + return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); } @Override - public ArgumentEntry copy() { - return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { + return updateTsRollingEntry(tsRollingEntry); + } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { + return updateSingleValueEntry(singleValueEntry); + } else { + throw new IllegalArgumentException("Unsupported argument entry type for rolling argument entry: " + entry.getType()); + } + } + + private boolean updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { + boolean updated = false; + for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { + updated |= addTsRecordIfAbsent(tsRecordEntry.getKey(), tsRecordEntry.getValue()); + } + return updated; } - public void addTsRecord(Long key, Object value) { + private boolean updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { + return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getValue()); + } + + private boolean addTsRecordIfAbsent(Long ts, Object value) { + if (!tsRecords.containsKey(ts)) { + addTsRecord(ts, value); + return true; + } + return false; + } + + private void addTsRecord(Long ts, Object value) { if (NumberUtils.isParsable(value.toString())) { - tsRecords.put(key, value); + tsRecords.put(ts, value); if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { tsRecords.pollFirstEntry(); } } else { - log.warn("Argument type 'TS_ROLLING' only supports numeric values."); - } - } - - public void addAllTsRecords(Map newRecords) { - for (Map.Entry entry : newRecords.entrySet()) { - addTsRecord(entry.getKey(), entry.getValue()); + throw new IllegalArgumentException("Argument type " + getType() + " only supports numeric values."); } } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java new file mode 100644 index 0000000000..cbaea6575c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -0,0 +1,238 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(classes = DefaultTbelInvokeService.class) +public class ScriptCalculatedFieldStateTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); + private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); + + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 43, 122L); + private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); + + private final long ts = System.currentTimeMillis(); + + private ScriptCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @Autowired + private TbelInvokeService tbelInvokeService; + + @BeforeEach + void setUp() { + ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService); + state = new ScriptCalculatedFieldState(ctx.getArgNames()); + } + + @Test + void testType() { + assertThat(state.getType()).isEqualTo(CalculatedFieldType.SCRIPT); + } + + @Test + void testUpdateState() { + state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry)); + + Map newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "assetHumidity", assetHumidityArgEntry, + "deviceTemperature", deviceTemperatureArgEntry + ) + ); + } + + @Test + void testUpdateStateWhenUpdateExistingEntry() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, 41, 349L); + Map newArgs = Map.of("assetHumidity", newArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "assetHumidity", newArgEntry, + "deviceTemperature", deviceTemperatureArgEntry + ) + ); + } + + @Test + void testPerformCalculation() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43)); + } + + @Test + void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); + + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, 4);// will not be used for calculation + values.put(ts - 45000, 2);// will not be used for calculation + values.put(ts - 20, 0); + + argumentEntry.setTsRecords(values); + + state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + } + + @Test + void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 1000);// will not be used + values.put(ts - 18, 0); + values.put(ts - 16, 0); + values.put(ts - 14, 0); + values.put(ts - 12, 0); + values.put(ts - 10, 0); + argumentEntry.setTsRecords(values); + + state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + } + + @Test + void testIsReadyWhenNotAllArgPresent() { + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenAllArgPresent() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + assertThat(state.isReady()).isTrue(); + } + + @Test + void testIsReadyWhenEmptyEntryPresents() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry)); + + assertThat(state.isReady()).isFalse(); + } + + private TsRollingArgumentEntry createRollingArgEntry() { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); + long ts = System.currentTimeMillis(); + + TreeMap values = new TreeMap<>(); + values.put(ts - 40, 10); + values.put(ts - 30, 12); + values.put(ts - 20, 17); + + argumentEntry.setTsRecords(values); + return argumentEntry; + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(ASSET_ID); + calculatedField.setType(CalculatedFieldType.SCRIPT); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig()); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig() { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + argument1.setRefEntityId(DEVICE_ID); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temperature", ArgumentType.TS_ROLLING, null); + argument1.setRefEntityKey(refEntityKey1); + argument1.setLimit(5); + argument1.setTimeWindow(30000); + + Argument argument2 = new Argument(); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey2); + + config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); + + config.setExpression("var result = 0; foreach(element : deviceTemperature.entrySet()) { result += element.getValue(); } var map = {}; map.put(\"averageDeviceTemperature\", result / deviceTemperature.size()); map.put(\"assetHumidity\", assetHumidity); return map;"); + + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + return config; + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java new file mode 100644 index 0000000000..58a981824c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -0,0 +1,213 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class SimpleCalculatedFieldStateTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); + private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); + + private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 11, 145L); + private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, 15, 165L); + private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, 23, 184L); + + private SimpleCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @BeforeEach + void setUp() { + ctx = new CalculatedFieldCtx(getCalculatedField(), null); + state = new SimpleCalculatedFieldState(ctx.getArgNames()); + } + + @Test + void testType() { + assertThat(state.getType()).isEqualTo(CalculatedFieldType.SIMPLE); + } + + @Test + void testUpdateState() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + + Map newArgs = Map.of("key3", key3ArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + ) + ); + } + + @Test + void testUpdateStateWhenUpdateExistingEntry() { + state.arguments = new HashMap<>(Map.of("key1", key1ArgEntry)); + + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), 18, 190L); + Map newArgs = Map.of("key1", newArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry)); + } + + @Test + void testUpdateStateWhenRollingEntryPassed() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + + Map newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY); + assertThatThrownBy(() -> state.updateState(newArgs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Rolling argument entry is not supported for simple calculated fields."); + } + + @Test + void testPerformCalculation() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + )); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("output", 49.0)); + } + + @Test + void testPerformCalculationWhenPassedNotNumber() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, "string", 124L), + "key3", key3ArgEntry + )); + + assertThatThrownBy(() -> state.performCalculation(ctx)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument 'key2' is not a number."); + } + + @Test + void testIsReadyWhenNotAllArgPresent() { + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenAllArgPresent() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + )); + + assertThat(state.isReady()).isTrue(); + } + + @Test + void testIsReadyWhenEmptyEntryPresents() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY); + + assertThat(state.isReady()).isFalse(); + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig()); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig() { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + argument1.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temp1", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey1); + + Argument argument2 = new Argument(); + argument2.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("temp2", ArgumentType.ATTRIBUTE, null); + argument2.setRefEntityKey(refEntityKey2); + + Argument argument3 = new Argument(); + argument3.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey3 = new ReferencedEntityKey("temp3", ArgumentType.TS_LATEST, null); + argument3.setRefEntityKey(refEntityKey3); + + config.setArguments(Map.of("key1", argument1, "key2", argument2, "key3", argument3)); + + config.setExpression("key1 + key2 + key3"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + return config; + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java new file mode 100644 index 0000000000..285da0b423 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class SingleValueArgumentEntryTest { + + private SingleValueArgumentEntry entry; + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + entry = new SingleValueArgumentEntry(ts, 11, 363L); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.SINGLE_VALUE); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWithThaSameTs() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, 13, 363L))).isFalse(); + } + + @Test + void testUpdateEntryWhenNewVersionIsNull() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, 13, null))).isTrue(); + assertThat(entry.getValue()).isEqualTo(13); + assertThat(entry.getVersion()).isNull(); + } + + @Test + void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 369L))).isTrue(); + assertThat(entry.getValue()).isEqualTo(18); + assertThat(entry.getVersion()).isEqualTo(369L); + } + + @Test + void testUpdateEntryWhenNewVersionIsLessThanCurrent() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 234L))).isFalse(); + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java new file mode 100644 index 0000000000..b08c5f2a58 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.TreeMap; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TsRollingArgumentEntryTest { + + private TsRollingArgumentEntry entry; + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + TreeMap values = new TreeMap<>(); + values.put(ts - 40, 10); + values.put(ts - 30, 12); + values.put(ts - 20, 17); + + entry = new TsRollingArgumentEntry(values); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenSingleValueEntryPassed() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, 23, 123L); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(4); + assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23); + } + + @Test + void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, 23, 123L); + + assertThat(entry.updateEntry(newEntry)).isFalse(); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 16); + values.put(ts - 10, 7); + values.put(ts - 5, 1); + newEntry.setTsRecords(values); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(5); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 40, 10, + ts - 30, 12, + ts - 20, 17, + ts - 10, 7, + ts - 5, 1 + )); + } + + @Test + void testUpdateEntryWhenValueIsNotNumber() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, "string", 123L); + + assertThatThrownBy(() -> entry.updateEntry(newEntry)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument type " + ArgumentEntryType.TS_ROLLING + " only supports numeric values."); + } + +} \ No newline at end of file From 5aabbd0f1ee683d99dffcde7325f0eb7253dbd05 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 29 Jan 2025 15:37:36 +0200 Subject: [PATCH 139/416] Link dispatch implementation --- .../CalculatedFieldEntityActor.java | 5 +- ...CalculatedFieldEntityMessageProcessor.java | 59 ++++++- .../CalculatedFieldManagerActor.java | 2 + ...alculatedFieldManagerMessageProcessor.java | 125 +++++++++++++- ...tityCalculatedFieldLinkedTelemetryMsg.java | 42 +++++ .../calculatedField/MultipleTbCallback.java | 15 +- .../cf/CalculatedFieldExecutionService.java | 6 +- ...efaultCalculatedFieldExecutionService.java | 159 ++++++++++-------- .../cf/ctx/state/CalculatedFieldCtx.java | 11 ++ ...faultTbCalculatedFieldConsumerService.java | 7 +- .../queue/DefaultTbClusterService.java | 13 ++ .../server/service/queue/TbPackCallback.java | 3 + .../server/cluster/TbClusterService.java | 3 + .../server/common/msg/queue/TbCallback.java | 8 + common/proto/src/main/proto/queue.proto | 3 +- 15 files changed, 365 insertions(+), 96 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index 43f057ad01..a5461c6152 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -43,7 +43,7 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); } catch (Exception e) { log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); - throw new TbActorException("Failed to initialize device actor", e); + throw new TbActorException("Failed to initialize CF entity actor", e); } } @@ -56,6 +56,9 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { case CF_ENTITY_TELEMETRY_MSG: processor.process((EntityCalculatedFieldTelemetryMsg) msg); break; + case CF_LINKED_TELEMETRY_MSG: + processor.process((EntityCalculatedFieldLinkedTelemetryMsg) msg); + break; default: return false; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 533eeb5e31..cea3cce791 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -18,15 +18,19 @@ package org.thingsboard.server.actors.calculatedField; import com.google.common.util.concurrent.ListenableFuture; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -40,6 +44,8 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -96,7 +102,25 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Set cfIds, List cfIdList, MultipleTbCallback callback) { + public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) { + var proto = msg.getProto(); + var ctx = msg.getCtx(); + var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); + List cfIds = getCalculatedFieldIds(proto); + if (cfIds.contains(ctx.getCfId())) { + callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList())); + } else if (proto.getAttrDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList())); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } + + private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection cfIds, List cfIdList, MultipleTbCallback callback) { if (cfIds.contains(ctx.getCfId())) { callback.onSuccess(CALLBACKS_PER_CF); } else { @@ -120,8 +144,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList())); } + @SneakyThrows private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, - Map newArgValues) throws InterruptedException, ExecutionException, TimeoutException { + Map newArgValues) { if (newArgValues.isEmpty()) { callback.onSuccess(CALLBACKS_PER_CF); } @@ -159,8 +184,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { + return mapToArguments(ctx.getMainEntityArguments(), data); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if(argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArguments(argNames, data); + } + + private static Map mapToArguments(Map argNames, List data) { + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } Map arguments = new HashMap<>(); - var argNames = ctx.getMainEntityArguments(); for (TsKvProto item : data) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); String argName = argNames.get(key); @@ -177,8 +216,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { + return mapToArguments(ctx.getMainEntityArguments(), scope, attrDataList); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if(argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArguments(argNames, scope, attrDataList); + } + + private static Map mapToArguments(Map argNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); - var argNames = ctx.getMainEntityArguments(); for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = argNames.get(key); @@ -196,4 +246,5 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } return cfIds; } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 87292d3206..1c198c660d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -73,6 +73,8 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); break; case CF_LINKED_TELEMETRY_MSG: + processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); + break; case CF_ENTITY_UPDATE_MSG: // processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4bc0589e95..4bce7cb322 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,29 +16,52 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -50,18 +73,22 @@ import java.util.concurrent.CopyOnWriteArrayList; @Slf4j public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { - private final Map calculatedFields = new HashMap<>(); + private final Map calculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); + private final CalculatedFieldExecutionService cfService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; protected TbActorCtx ctx; final TenantId tenantId; + private final static int initFetchPackSize = 1024; CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); + this.cfService = systemContext.getCalculatedFieldExecutionService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.tenantId = tenantId; @@ -73,11 +100,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onFieldInitMsg(CalculatedFieldInitMsg msg) { var cf = msg.getCf(); - calculatedFields.put(cf.getId(), cf); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()) - .add(new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService())); + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); msg.getCallback().onSuccess(); } @@ -99,14 +126,70 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); - var proto = msg.getProto(); + // 2 = 1 for CF processing + 1 for links processing + MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); // process all cfs related to entity, or it's profile; var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); - //TODO: Transfer only 'part' of the original callback. - getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, msg.getCallback())); + if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { + getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); + } else { + callback.onSuccess(); + } // process all links (if any); - var links = getCalculatedFieldLinksByEntityId(entityId); + List linkedCalculatedFields = filterCalculatedFieldLinks(msg); + var linksSize = linkedCalculatedFields.size(); + if (linksSize > 0) { + cfService.pushMsgToLinks(msg, linkedCalculatedFields, callback); + } else { + callback.onSuccess(); + } + } + + public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { + EntityId sourceEntityId = msg.getEntityId(); + var proto = msg.getProto(); + var linksList = proto.getLinksList(); + for (var linkProto : linksList) { + var link = toCalculatedFieldEntityCtxId(linkProto); + var targetEntityId = link.entityId(); + var targetEntityType = targetEntityId.getEntityType(); + var cf = calculatedFields.get(link.cfId()); + if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { + // iterate over all entities that belong to profile and push the message for corresponding CF + var entityIds = getEntitiesByProfile(targetEntityId); + if (!entityIds.isEmpty()) { + MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); + entityIds.forEach(entityId -> getOrCreateActor(entityId).tell(newMsg)); + } else { + msg.getCallback().onSuccess(); + } + } else { + // push the message to specific entity; + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); + getOrCreateActor(targetEntityId).tell(newMsg); + } + } + } + + private CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId(CalculatedFieldEntityCtxIdProto ctxIdProto) { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + private List filterCalculatedFieldLinks(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + var proto = msg.getProto(); + List result = new ArrayList<>(); + for (var link : getCalculatedFieldLinksByEntityId(entityId)) { + CalculatedFieldCtx ctx = calculatedFields.get(link.getCalculatedFieldId()); + if (ctx.linkMatches(entityId, proto)) { + result.add(ctx.toCalculatedFieldEntityCtxId()); + } + } + return result; } private List getCalculatedFieldsByEntityId(EntityId entityId) { @@ -131,6 +214,30 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } + private Set getEntitiesByProfile(EntityId entityProfileId) { + Set entities = profileEntities.get(entityProfileId); + if (entities == null) { + entities = switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + systemContext.getAssetService().findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + systemContext.getDeviceService().findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); + return entities; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); @@ -139,7 +246,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware }; } - protected TbActorRef getOrCreateActor(EntityId entityId) { + private TbActorRef getOrCreateActor(EntityId entityId) { return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), () -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, () -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java new file mode 100644 index 0000000000..47b91cbe65 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityCalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINKED_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java index 18e700f38c..312cf72bed 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java @@ -15,35 +15,42 @@ */ package org.thingsboard.server.actors.calculatedField; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; +@Slf4j public class MultipleTbCallback implements TbCallback { - + @Getter + private final UUID id; private final AtomicInteger counter; private final TbCallback callback; public MultipleTbCallback(int count, TbCallback callback) { + id = UUID.randomUUID(); this.counter = new AtomicInteger(count); this.callback = callback; } @Override public void onSuccess() { - if (counter.decrementAndGet() <= 0) { - callback.onSuccess(); - } + onSuccess(1); } public void onSuccess(int number) { + log.trace("[{}][{}] onSuccess({})", id, callback.getId(), number); if (counter.addAndGet(-number) <= 0) { + log.trace("[{}][{}] Done.", id, callback.getId()); callback.onSuccess(); } } @Override public void onFailure(Throwable t) { + log.warn("[{}][{}] onFailure.", id, callback.getId()); callback.onFailure(t); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 806c224608..d8daae2e64 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -53,6 +54,10 @@ public interface CalculatedFieldExecutionService { ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); + + void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); + // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ @@ -65,6 +70,5 @@ public interface CalculatedFieldExecutionService { void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); - void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f333eecbd8..19561b73b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -34,6 +34,8 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; +import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; @@ -83,6 +85,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -636,60 +639,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } -// private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { -// CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); -// Map argumentsMap = new HashMap<>(argumentValues); -// -// CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); -// -// states.compute(entityCtxId, (ctxId, ctx) -> { -// CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); -// -// CompletableFuture updateFuture = new CompletableFuture<>(); -// -// Consumer performUpdateState = (state) -> { -// if (state.updateState(argumentsMap)) { -// calculatedFieldEntityCtx.setState(state); -// stateService.persistState(entityCtxId, calculatedFieldEntityCtx); -// Map arguments = state.getArguments(); -// boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && -// !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); -// if (allArgsPresent) { -// performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); -// } -// log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); -// } -// updateFuture.complete(null); -// }; -// -// CalculatedFieldState state = calculatedFieldEntityCtx.getState(); -// -// boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); -// boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() -// .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); -// -// if (!allKeysPresent || requiresTsRollingUpdate) { -// Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() -// .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) -// .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); -// -// fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) -// .addListener(() -> performUpdateState.accept(state), -// calculatedFieldCallbackExecutor); -// } else { -// performUpdateState.accept(state); -// } -// -// try { -// updateFuture.join(); -// } catch (Exception e) { -// log.trace("Failed to update state for ctxId [{}].", ctxId, e); -// throw new RuntimeException("Failed to update or initialize state.", e); -// } -// -// return calculatedFieldEntityCtx; -// }); -// } @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { @@ -713,9 +662,74 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); } catch (Exception e) { log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); + callback.onFailure(e); } } + @Override + public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback) { + Map> unicasts = new HashMap<>(); + List broadcasts = new ArrayList<>(); + for (CalculatedFieldEntityCtxId link : linkedCalculatedFields) { + var linkEntityId = link.entityId(); + var linkEntityType = linkEntityId.getEntityType(); + // Let's assume number of entities in profile is N, and number of partitions is P. If N > P, we save by broadcasting to all partitions. Usually N >> P. + boolean broadcast = EntityType.DEVICE_PROFILE.equals(linkEntityType) || EntityType.ASSET_PROFILE.equals(linkEntityType); + if (broadcast) { + broadcasts.add(link); + } else { + TopicPartitionInfo tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, link.entityId()); + unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); + } + } + MultipleTbCallback linkCallback = new MultipleTbCallback(2, callback); + if (!broadcasts.isEmpty()) { + broadcast(broadcasts, msg, linkCallback); + } else { + linkCallback.onSuccess(); + } + if (!unicasts.isEmpty()) { + unicast(unicasts, msg, linkCallback); + } else { + linkCallback.onSuccess(); + } + } + + private void unicast(Map> unicasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { + TbQueueCallback callback = new TbCallbackWrapper(new MultipleTbCallback(unicasts.size(), mainCallback)); + unicasts.forEach((topicPartitionInfo, ctxIds) -> { + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), + ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); + }); + } + + private void broadcast(List broadcasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { + TbQueueCallback callback = new TbCallbackWrapper(mainCallback); + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), broadcasts); + clusterService.broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); + } + + private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { + TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + builder.setMsg(telemetryProto); + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + return builder.build(); + } + + //TODO: IM: move to utils; + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { Map argumentValues = new HashMap<>(); List> futures = new ArrayList<>(); @@ -868,25 +882,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return telemetryMsg; } - private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { - TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); - builder.setMsg(telemetryProto); - for (CalculatedFieldEntityCtxId link : links) { - builder.addLinks(toProto(link)); - } - return builder.build(); - } - - private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { return TransportProtos.CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) @@ -948,4 +943,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private static class TbCallbackWrapper implements TbQueueCallback { + private final TbCallback callback; + + public TbCallbackWrapper(TbCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 1a0a16254a..37e402fbc8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -31,7 +31,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; import java.util.HashMap; @@ -143,4 +145,13 @@ public class CalculatedFieldCtx { } return false; } + + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { + //TODO: IM - implement + return true; + } + + public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { + return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 0ccecbbeca..0fcc04bd5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -223,15 +223,16 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); if (toCfNotification.hasComponentLifecycle()) { - // from upstream (maybe removed since we dont need to init state for each partition) + // from upstream (maybe removed since we don't need to init state for each partition) forwardToActorSystem(toCfNotification.getComponentLifecycle(), callback); handleComponentLifecycleMsg(id, ProtoUtils.fromProto(toCfNotification.getComponentLifecycle())); } else if (toCfNotification.hasEntityUpdateMsg()) { processEntityUpdateMsg(toCfNotification.getEntityUpdateMsg()); - // from upstream (maybe removed since we dont need to update state for each partition) + // from upstream (maybe removed since we don't need to update state for each partition) forwardToActorSystem(toCfNotification.getEntityUpdateMsg(), callback); + } else if (toCfNotification.hasLinkedTelemetryMsg()) { + forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); } - callback.onSuccess(); } private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index b1a4b1859e..2783e826fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -194,6 +194,19 @@ public class DefaultTbClusterService implements TbClusterService { } } + @Override + public void broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg toCfMsg, TbQueueCallback callback) { + UUID msgId = UUID.randomUUID(); + TbQueueProducer> toCfProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); + Set tbReServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + for (String serviceId : tbReServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + toCfProducer.send(tpi, new TbProtoQueueMsg<>(msgId, toCfMsg), null); + toRuleEngineNfs.incrementAndGet(); + } + callback.onSuccess(null); // TODO: refactor to be fair, similar to multi-value callback; + } + @Override public void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, TenantId.SYS_TENANT_ID, tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index f9d15353c7..3402a357d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.queue; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -23,9 +24,11 @@ import java.util.UUID; @Slf4j public class TbPackCallback implements TbCallback { private final TbPackProcessingContext ctx; + @Getter private final UUID id; public TbPackCallback(UUID id, TbPackProcessingContext ctx) { + log.trace("[{}] CALLBACK CREATED", id); this.id = id; this.ctx = ctx; } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 269db0a73a..588b135891 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -62,6 +63,8 @@ public interface TbClusterService extends TbQueueClusterService { void broadcastToCore(ToCoreNotificationMsg msg); + void broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg build, TbQueueCallback callback); + void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback); void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java index a2d264a4ff..f8204fb0ae 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java @@ -15,6 +15,10 @@ */ package org.thingsboard.server.common.msg.queue; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.UUID; + public interface TbCallback { TbCallback EMPTY = new TbCallback() { @@ -30,6 +34,10 @@ public interface TbCallback { } }; + default UUID getId(){ + return EntityId.NULL_UUID; + } + void onSuccess(); void onFailure(Throwable t); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 9e3e842cd9..69b912120e 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -803,7 +803,7 @@ message CalculatedFieldTelemetryMsgProto { message CalculatedFieldLinkedTelemetryMsgProto { CalculatedFieldTelemetryMsgProto msg = 1; - repeated CalculatedFieldEntityCtxIdProto links = 2; + repeated CalculatedFieldEntityCtxIdProto links = 2; } message CalculatedFieldEntityCtxIdProto { @@ -1639,6 +1639,7 @@ message ToCalculatedFieldMsg { message ToCalculatedFieldNotificationMsg { ComponentLifecycleMsgProto componentLifecycle = 1; CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 2; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; } /* Messages that are handled by ThingsBoard RuleEngine Service */ From 8b7bf4a633c70811e5e36746e728c3b4a003fddf Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 29 Jan 2025 18:43:15 +0200 Subject: [PATCH 140/416] UI: Add new 'Get Dashboard State with Parameters' action type --- .../widget/lib/action/action-widget.models.ts | 23 +++++++++++++++++++ ...value-action-settings-panel.component.html | 4 +++- ...t-value-action-settings-panel.component.ts | 10 +++++--- .../get-value-action-settings.component.ts | 8 +++++++ .../models/action-widget-settings.models.ts | 6 +++-- .../assets/locale/locale.constant-en_US.json | 3 +++ 6 files changed, 48 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts index b4a46d9592..aa001f5a1b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts @@ -57,6 +57,7 @@ import { isDefinedAndNotNull } from '@core/utils'; import { parseError } from '@shared/models/error.models'; import { CompiledTbFunction, compileTbFunction } from '@shared/models/js-function.models'; import { HttpClient } from '@angular/common/http'; +import { StateObject } from '@core/api/widget-api.models'; @Directive() // eslint-disable-next-line @angular-eslint/directive-class-suffix @@ -269,6 +270,8 @@ export abstract class ValueGetter extends ValueAction { return new AlarmStatusValueGetter(ctx, settings, valueType, valueObserver, simulated); case GetValueAction.GET_DASHBOARD_STATE: return new DashboardStateGetter(ctx, settings, valueType, valueObserver, simulated); + case GetValueAction.GET_DASHBOARD_STATE_WITH_PARAMS: + return new DashboardStateWithParamsGetter(ctx, settings, valueType, valueObserver, simulated); } } @@ -641,6 +644,26 @@ export class DashboardStateGetter extends ValueGetter { } } +export class DashboardStateWithParamsGetter extends ValueGetter { + constructor(protected ctx: WidgetContext, + protected settings: GetValueSettings, + protected valueType: ValueType, + protected valueObserver: Partial>, + protected simulated: boolean) { + super(ctx, settings, valueType, valueObserver, simulated); + } + + protected doGetValue(): Observable { + if (this.simulated) { + return of({id: 'default', params: {}}); + } else { + return this.ctx.stateController.dashboardCtrl.dashboardCtx.stateId.pipe( + map(id => ({id, params: this.ctx.stateController.getStateParams()})) + ); + } + } +} + export class ExecuteRpcValueSetter extends ValueSetter { private readonly executeRpcSettings: RpcSettings; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.html index cb6cd5cbe0..68a9872799 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.html @@ -131,7 +131,9 @@ class="tb-form-panel stroked" formGroupName="dataToValue">
widgets.value-action.action-result-converter
- + {{ 'widgets.value-action.converter-none' | translate }} {{ 'widgets.value-action.converter-function' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.ts index a4339b7e12..82bc659e12 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.ts @@ -99,6 +99,8 @@ export class GetValueActionSettingsPanelComponent extends PageComponent implemen getValueSettingsFormGroup: UntypedFormGroup; + entityType = EntityType; + alarmSeverities = Object.keys(AlarmSeverity) as AlarmSeverity[]; alarmSeverityTranslationMap = alarmSeverityTranslations; @@ -167,7 +169,7 @@ export class GetValueActionSettingsPanelComponent extends PageComponent implemen private updateValidators() { const action: GetValueAction = this.getValueSettingsFormGroup.get('action').value; - const dataToValueType: DataToValueType = this.getValueSettingsFormGroup.get('dataToValue').get('type').value; + let dataToValueType: DataToValueType = this.getValueSettingsFormGroup.get('dataToValue').get('type').value; this.getValueSettingsFormGroup.get('defaultValue').disable({emitEvent: false}); this.getValueSettingsFormGroup.get('executeRpc').disable({emitEvent: false}); @@ -196,6 +198,10 @@ export class GetValueActionSettingsPanelComponent extends PageComponent implemen case GetValueAction.GET_ALARM_STATUS: this.getValueSettingsFormGroup.get('getAlarmStatus').enable({emitEvent: false}); break; + case GetValueAction.GET_DASHBOARD_STATE_WITH_PARAMS: + this.getValueSettingsFormGroup.get('dataToValue.type').setValue(DataToValueType.FUNCTION, {emitEvent: false}); + dataToValueType = DataToValueType.FUNCTION; + break } if (action === GetValueAction.DO_NOTHING || action === GetValueAction.GET_ALARM_STATUS) { this.getValueSettingsFormGroup.get('dataToValue').disable({emitEvent: false}); @@ -208,6 +214,4 @@ export class GetValueActionSettingsPanelComponent extends PageComponent implemen } } } - - protected readonly entityType = EntityType; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings.component.ts index 871923e5b7..d482d2e9fe 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings.component.ts @@ -191,6 +191,14 @@ export class GetValueActionSettingsComponent implements OnInit, ControlValueAcce this.displayValue = this.translate.instant('widgets.value-action.get-dashboard-state-text'); } break; + case GetValueAction.GET_DASHBOARD_STATE_WITH_PARAMS: + if (this.valueType === ValueType.BOOLEAN) { + const state = this.modelValue.dataToValue?.compareToValue; + this.displayValue = this.translate.instant('widgets.value-action.when-dashboard-state-with-params-function-is-text', {state}); + } else { + this.displayValue = this.translate.instant('widgets.value-action.get-dashboard-state-with-params-text'); + } + break; } this.cd.markForCheck(); } diff --git a/ui-ngx/src/app/shared/models/action-widget-settings.models.ts b/ui-ngx/src/app/shared/models/action-widget-settings.models.ts index b1c47a1fd3..1c83669206 100644 --- a/ui-ngx/src/app/shared/models/action-widget-settings.models.ts +++ b/ui-ngx/src/app/shared/models/action-widget-settings.models.ts @@ -25,7 +25,8 @@ export enum GetValueAction { GET_ATTRIBUTE = 'GET_ATTRIBUTE', GET_TIME_SERIES = 'GET_TIME_SERIES', GET_ALARM_STATUS = 'GET_ALARM_STATUS', - GET_DASHBOARD_STATE = 'GET_DASHBOARD_STATE' + GET_DASHBOARD_STATE = 'GET_DASHBOARD_STATE', + GET_DASHBOARD_STATE_WITH_PARAMS = 'GET_DASHBOARD_STATE_WITH_PARAMS', } export const getValueActions = Object.keys(GetValueAction) as GetValueAction[]; @@ -45,7 +46,8 @@ export const getValueActionTranslations = new Map( [GetValueAction.GET_ATTRIBUTE, 'widgets.value-action.get-attribute'], [GetValueAction.GET_TIME_SERIES, 'widgets.value-action.get-time-series'], [GetValueAction.GET_ALARM_STATUS, 'widgets.value-action.get-alarm-status'], - [GetValueAction.GET_DASHBOARD_STATE, 'widgets.value-action.get-dashboard-state'] + [GetValueAction.GET_DASHBOARD_STATE, 'widgets.value-action.get-dashboard-state'], + [GetValueAction.GET_DASHBOARD_STATE_WITH_PARAMS, 'widgets.value-action.get-dashboard-state-with-params'], ] ); diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index a16625bdaa..8e7e6deb7e 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -8265,14 +8265,17 @@ "get-time-series": "Get time series", "get-alarm-status": "Get alarm status", "get-dashboard-state": "Get dashboard state", + "get-dashboard-state-with-params": "Get dashboard state with params", "add-time-series": "Add time series", "execute-rpc-text": "Execute RPC method '{{methodName}}'", "get-time-series-text": "Use time series '{{key}}'", "get-attribute-text": "Use attribute '{{key}}'", "get-alarm-status-text": "Use alarm status", "get-dashboard-state-text": "Use dashboard state", + "get-dashboard-state-with-params-text": "Use dashboard state with params", "when-dashboard-state-is-text": "When dashboard state is '{{state}}'", "when-dashboard-state-function-is-text": "When f(dashboard state) is '{{state}}'", + "when-dashboard-state-with-params-function-is-text": "When f(dashboard state with params) is '{{state}}'", "set-attribute-to-value-text": "Set '{{key}}' attribute to: {{value}}", "add-time-series-value-text": "Add '{{key}}' time series value: {{value}}", "set-attribute-text": "Set '{{key}}' attribute", From c6bb337d23460f7b468085412d099ac08d6d09e9 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 29 Jan 2025 19:08:11 +0200 Subject: [PATCH 141/416] UI: Add help to function 'Get Dashboard State with Parameters' --- ...value-action-settings-panel.component.html | 2 +- ...t-value-action-settings-panel.component.ts | 8 ++++ ...alue_get_dashboard_state_with_params_fn.md | 40 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 ui-ngx/src/assets/help/en_US/widget/config/parse_value_get_dashboard_state_with_params_fn.md diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.html index 68a9872799..7783c0af20 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.html @@ -145,7 +145,7 @@ [globalVariables]="functionScopeVariables" [functionArgs]="['data']" functionTitle="{{ 'widgets.value-action.parse-value-function' | translate }}" - helpId="widget/lib/rpc/parse_value_fn"> + [helpId]="getParseValueFunctionHelpId()">
{{ 'widgets.value-action.state-when-result-is' | translate:{state: stateLabel} }}
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.ts index 82bc659e12..4c98c0cac4 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/get-value-action-settings-panel.component.ts @@ -167,6 +167,14 @@ export class GetValueActionSettingsPanelComponent extends PageComponent implemen this.getValueSettingsApplied.emit(getValueSettings); } + getParseValueFunctionHelpId(): string { + const action: GetValueAction = this.getValueSettingsFormGroup.get('action').value; + if (action === GetValueAction.GET_DASHBOARD_STATE_WITH_PARAMS) { + return 'widget/config/parse_value_get_dashboard_state_with_params_fn'; + } + return 'widget/lib/rpc/parse_value_fn'; + } + private updateValidators() { const action: GetValueAction = this.getValueSettingsFormGroup.get('action').value; let dataToValueType: DataToValueType = this.getValueSettingsFormGroup.get('dataToValue').get('type').value; diff --git a/ui-ngx/src/assets/help/en_US/widget/config/parse_value_get_dashboard_state_with_params_fn.md b/ui-ngx/src/assets/help/en_US/widget/config/parse_value_get_dashboard_state_with_params_fn.md new file mode 100644 index 0000000000..c272a07f79 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/widget/config/parse_value_get_dashboard_state_with_params_fn.md @@ -0,0 +1,40 @@ +#### Parse value function + +
+
+ +*function (data): boolean* + +A JavaScript function that converts the current dashboard state and parameters into a boolean value. + +**Parameters:** + +
    +
  • data: StateObject - the current dashboard state object. +
    • +
+ +**Returns:** + +`true` if the widget should be in an activated state, `false` otherwise. + +
+ +##### Examples + +* Check if the current dashboard state ID is "default": + +```javascript +return data.id === 'default' ? true : false; +{:copy-code} +``` + +* Check if the current dashboard state parameters are empty: + +```javascript +return Object.keys(data.params).length ? true : false; +{:copy-code} +``` + +
+
From a4aa2444acf62026d11f4601076885ba2ccc213c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 30 Jan 2025 11:01:45 +0200 Subject: [PATCH 142/416] added new endpoint --- .../controller/CalculatedFieldController.java | 35 ++++++++++--------- .../cf/DefaultTbCalculatedFieldService.java | 9 +++++ .../entitiy/cf/TbCalculatedFieldService.java | 5 +++ .../server/dao/cf/CalculatedFieldService.java | 2 ++ .../dao/cf/BaseCalculatedFieldService.java | 8 +++++ .../server/dao/cf/CalculatedFieldDao.java | 2 ++ .../dao/sql/cf/CalculatedFieldRepository.java | 4 +++ ...efaultNativeCalculatedFieldRepository.java | 2 +- .../dao/sql/cf/JpaCalculatedFieldDao.java | 6 ++++ 9 files changed, 56 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index efe25e3883..124dd8d710 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -32,6 +32,8 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.config.annotations.ApiOperation; @@ -40,7 +42,8 @@ import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; @@ -91,25 +94,25 @@ public class CalculatedFieldController extends BaseController { return calculatedField; } - @ApiOperation(value = "Get Calculated Fields (getCalculatedFields)", - notes = "Returns a page of calculated fields. " + PAGE_DATA_PARAMETERS + @ApiOperation(value = "Get Calculated Fields (getCalculatedFieldsByEntityId)", + notes = "Fetch the Calculated Fields based on the provided Entity Id." ) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) + @RequestMapping(value = "/{entityType}/{entityId}/calculatedField", params = {"pageSize", "page"}, method = RequestMethod.GET) @ResponseBody - public PageData getCalculatedFields( - @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) - @RequestParam int pageSize, - @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page, - @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) - @RequestParam(required = false) String textSearch, - @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) - @RequestParam(required = false) String sortProperty, - @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) - @RequestParam(required = false) String sortOrder) throws ThingsboardException { + public PageData getCalculatedFieldsByEntityId( + @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, + @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, + @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, + @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) @RequestParam int page, + @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) @RequestParam(required = false) String textSearch, + @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) @RequestParam(required = false) String sortProperty, + @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) @RequestParam(required = false) String sortOrder) throws ThingsboardException { PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); - return checkNotNull(calculatedFieldService.findAllCalculatedFields(pageLink)); + checkParameter("entityId", entityIdStr); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityIdStr); + checkEntityId(entityId, Operation.READ_CALCULATED_FIELD); + return checkNotNull(tbCalculatedFieldService.findAllByTenantIdAndEntityId(entityId, getCurrentUser(), pageLink)); } @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 2e6e975636..c8c589691b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; @@ -74,6 +76,13 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); } + @Override + public PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink) { + TenantId tenantId = user.getTenantId(); + checkEntityExistence(tenantId, entityId); + return calculatedFieldService.findAllCalculatedFieldsByEntityId(tenantId, entityId, pageLink); + } + @Override @Transactional public void delete(CalculatedField calculatedField, SecurityUser user) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 89931b8541..cb7e809059 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,6 +18,9 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { @@ -26,6 +29,8 @@ public interface TbCalculatedFieldService { CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); + PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink); + void delete(CalculatedField calculatedField, SecurityUser user); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 3a508a5c08..f08d54190f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -44,6 +44,8 @@ public interface CalculatedFieldService extends EntityDaoService { PageData findAllCalculatedFields(PageLink pageLink); + PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 26ed4134cc..d37a4a53c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -118,6 +118,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findAll(pageLink); } + @Override + public PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + log.trace("Executing findAllByEntityId, entityId [{}], pageLink [{}]", entityId, pageLink); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + validatePageLink(pageLink); + return calculatedFieldDao.findAllByEntityId(tenantId, entityId, pageLink); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { validateId(tenantId, id -> INCORRECT_TENANT_ID + id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 39663d0afc..23a2eae93e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -37,6 +37,8 @@ public interface CalculatedFieldDao extends Dao { PageData findAll(PageLink pageLink); + PageData findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); boolean existsByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index bed6f2d3a2..c0118d4f02 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.cf; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -30,6 +32,8 @@ public interface CalculatedFieldRepository extends JpaRepository findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + Page findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, Pageable pageable); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 677234dc20..fcbf4d4dd0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -83,7 +83,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF CalculatedField calculatedField = new CalculatedField(); calculatedField.setId(new CalculatedFieldId(id)); calculatedField.setCreatedTime(createdTime); - calculatedField.setTenantId(new TenantId(tenantId)); + calculatedField.setTenantId(TenantId.fromUUID(tenantId)); calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); calculatedField.setType(type); calculatedField.setName(name); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index cdcffdd440..34bfa27b16 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -71,6 +71,12 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + log.debug("Try to find calculated fields by entityId[{}] and pageLink [{}]", entityId, pageLink); + return DaoUtil.toPageData(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { From 50f36c7a5ce7513ade46bbc934b39fec61783963 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 31 Jan 2025 10:11:47 +0200 Subject: [PATCH 143/416] UI: In action dashboard state params added new instance params --- .../home/components/widget/lib/action/action-widget.models.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts index aa001f5a1b..dbf83aac9a 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts @@ -53,7 +53,7 @@ import { import { ValueType } from '@shared/models/constants'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { EntityId } from '@shared/models/id/entity-id'; -import { isDefinedAndNotNull } from '@core/utils'; +import { deepClone, isDefinedAndNotNull } from '@core/utils'; import { parseError } from '@shared/models/error.models'; import { CompiledTbFunction, compileTbFunction } from '@shared/models/js-function.models'; import { HttpClient } from '@angular/common/http'; @@ -658,7 +658,7 @@ export class DashboardStateWithParamsGetter extends ValueGetter { return of({id: 'default', params: {}}); } else { return this.ctx.stateController.dashboardCtrl.dashboardCtx.stateId.pipe( - map(id => ({id, params: this.ctx.stateController.getStateParams()})) + map(id => ({id, params: deepClone(this.ctx.stateController.getStateParams())})) ); } } From fd42c51df1699c371350c264f5e7116df0ebda94 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 11:25:22 +0200 Subject: [PATCH 144/416] Calculated field add/edit basic implementation --- .../core/http/calculated-fields.service.ts | 56 +---- .../calculated-fields-table-config.ts | 74 ++++-- .../calculated-fields-table.component.ts | 23 +- ...lated-field-arguments-table.component.html | 106 +++++++++ ...lated-field-arguments-table.component.scss | 32 +++ ...culated-field-arguments-table.component.ts | 213 ++++++++++++++++++ .../calculated-field-dialog.component.html | 169 ++++++++++++++ .../calculated-field-dialog.component.ts | 122 ++++++++++ ...ulated-field-argument-panel.component.html | 198 ++++++++++++++++ ...ulated-field-argument-panel.component.scss | 29 +++ ...lculated-field-argument-panel.component.ts | 201 +++++++++++++++++ .../components/public-api.ts | 18 ++ .../home/components/home-components.module.ts | 19 +- .../pages/device/device-tabs.component.ts | 4 +- .../entity/entity-autocomplete.component.html | 13 +- .../entity/entity-autocomplete.component.ts | 7 + .../entity-key-autocomplete.component.html | 39 ++++ .../entity-key-autocomplete.component.ts | 134 +++++++++++ .../shared/components/js-func.component.ts | 46 ++-- .../shared/models/calculated-field.models.ts | 94 +++++++- ui-ngx/src/app/shared/models/public-api.ts | 1 + .../src/app/shared/models/regex.constants.ts | 17 ++ ui-ngx/src/app/shared/shared.module.ts | 7 +- .../assets/locale/locale.constant-en_US.json | 67 +++++- 24 files changed, 1572 insertions(+), 117 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts create mode 100644 ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html create mode 100644 ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts create mode 100644 ui-ngx/src/app/shared/models/regex.constants.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 5df4a84949..dca0c9a3c7 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -16,7 +16,7 @@ import { Injectable } from '@angular/core'; import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; -import { Observable, of } from 'rxjs'; +import { Observable } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; @@ -25,64 +25,26 @@ import { PageLink } from '@shared/models/page/page-link'; @Injectable({ providedIn: 'root' }) -// [TODO]: [Calculated fields] - implement when BE ready export class CalculatedFieldsService { - fieldsMock = [ - { - name: 'Calculated Field 1', - type: 'Simple', - configuration: { - expression: '1 + 2', - type: 'SIMPLE', - }, - entityId: '1', - id: { - id: '1', - } - }, - { - name: 'Calculated Field 2', - type: 'Script', - entityId: '2', - configuration: { - expression: '${power}', - type: 'SIMPLE', - }, - id: { - id: '2', - } - } - ] as any[]; - constructor( private http: HttpClient ) { } - public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { - return of(this.fieldsMock[0]); - // return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + public getCalculatedFieldById(calculatedFieldId: string, config?: RequestConfig): Observable { + return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { - return of(this.fieldsMock[1]); - // return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); + public saveCalculatedField(calculatedField: CalculatedField, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); } public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { - return of(true); - // return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public getCalculatedFields(pageLink: PageLink, - config?: RequestConfig): Observable> { - return of({ - data: this.fieldsMock, - totalPages: 1, - totalElements: 2, - hasNext: false, - }); - // return this.http.get>(`/api/calculatedField${pageLink.toQuery()}`, - // defaultHttpOptionsFromConfig(config)); + public getCalculatedFields(pageLink: PageLink, config?: RequestConfig): Observable> { + return this.http.get>(`/api/calculatedFields${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 22f742b9d8..2500c92336 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -23,41 +23,33 @@ import { TimePageLink } from '@shared/models/page/page-link'; import { Observable, of } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; -import { DialogService } from '@core/services/dialog.service'; import { MINUTE } from '@shared/models/time/time.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { getCurrentAuthState } from '@core/auth/auth.selectors'; -import { ChangeDetectorRef, DestroyRef, ViewContainerRef } from '@angular/core'; -import { Overlay } from '@angular/cdk/overlay'; -import { UtilsService } from '@core/services/utils.service'; -import { EntityService } from '@core/http/entity.service'; +import { getCurrentAuthState, getCurrentAuthUser } from '@core/auth/auth.selectors'; +import { DestroyRef } from '@angular/core'; import { EntityDebugSettings } from '@shared/models/entity.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; -import { catchError, switchMap } from 'rxjs/operators'; +import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField } from '@shared/models/calculated-field.models'; +import { CalculatedFieldDialogComponent } from './components/public-api'; export class CalculatedFieldsTableConfig extends EntityTableConfig { readonly calculatedFieldsDebugPerTenantLimitsConfiguration = getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; + readonly tenantId = getCurrentAuthUser(this.store).tenantId; constructor(private calculatedFieldsService: CalculatedFieldsService, - private entityService: EntityService, - private dialogService: DialogService, private translate: TranslateService, private dialog: MatDialog, public entityId: EntityId = null, private store: Store, - private viewContainerRef: ViewContainerRef, - private overlay: Overlay, - private cd: ChangeDetectorRef, - private utilsService: UtilsService, private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, private destroyRef: DestroyRef, @@ -67,6 +59,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); + this.addEntity = this.addCalculatedField.bind(this); + this.deleteEntityTitle = (field) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); + this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); + this.deleteEntitiesTitle = count => this.translate.instant('calculated-fields.delete-multiple-title', {count}); + this.deleteEntitiesContent = () => this.translate.instant('calculated-fields.delete-multiple-text'); + this.deleteEntity = id => this.calculatedFieldsService.deleteCalculatedField(id.id); this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; @@ -97,8 +96,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, - // // [TODO]: [Calculated fields] - implement edit - onAction: (_, entity) => {} + onAction: (_, entity) => this.editCalculatedField(entity) } ); } @@ -121,7 +119,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField} as any)), + ) + .subscribe((res) => { + if (res) { + this.updateData(); + } + }); + } + + private editCalculatedField(calculatedField: CalculatedField): void { + this.getCalculatedFieldDialog(calculatedField, 'action.apply') + .afterClosed() + .pipe( + filter(Boolean), + switchMap((updatedCalculatedField) => this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField} as any)), + ) + .subscribe((res) => { + if (res) { + this.updateData(); + } + }); + } + + private getCalculatedFieldDialog(value = {}, buttonTitle = 'action.add') { + return this.dialog.open(CalculatedFieldDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + value, + buttonTitle, + entityId: this.entityId, + debugLimitsConfiguration: this.calculatedFieldsDebugPerTenantLimitsConfiguration, + tenantId: this.tenantId, + } + }) + } + private getDebugConfigLabel(debugSettings: EntityDebugSettings): string { const isDebugActive = this.isDebugActive(debugSettings?.allEnabledUntil); @@ -149,7 +189,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ ...field, debugSettings })), catchError(() => of(null)), takeUntilDestroyed(this.destroyRef), diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index f98d24bd52..853870982a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -16,24 +16,18 @@ import { ChangeDetectionStrategy, - ChangeDetectorRef, Component, DestroyRef, Input, OnInit, ViewChild, - ViewContainerRef, } from '@angular/core'; import { EntityId } from '@shared/models/id/entity-id'; import { EntitiesTableComponent } from '@home/components/entity/entities-table.component'; -import { EntityService } from '@core/http/entity.service'; -import { DialogService } from '@core/services/dialog.service'; import { TranslateService } from '@ngx-translate/core'; import { MatDialog } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { Overlay } from '@angular/cdk/overlay'; -import { UtilsService } from '@core/services/utils.service'; import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/calculated-fields-table-config'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { TbPopoverService } from '@shared/components/popover.service'; @@ -51,7 +45,6 @@ export class CalculatedFieldsTableComponent implements OnInit { set entityId(entityId: EntityId) { if (this.entityIdValue !== entityId) { this.entityIdValue = entityId; - this.entitiesTable.resetSortAndFilter(this.activeValue); if (!this.activeValue) { this.hasInitialized = true; } @@ -78,18 +71,12 @@ export class CalculatedFieldsTableComponent implements OnInit { private entityIdValue: EntityId; constructor(private calculatedFieldsService: CalculatedFieldsService, - private entityService: EntityService, - private dialogService: DialogService, private translate: TranslateService, private dialog: MatDialog, private store: Store, - private overlay: Overlay, - private viewContainerRef: ViewContainerRef, - private cd: ChangeDetectorRef, private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, - private destroyRef: DestroyRef, - private utilsService: UtilsService) { + private destroyRef: DestroyRef) { } ngOnInit() { @@ -97,19 +84,13 @@ export class CalculatedFieldsTableComponent implements OnInit { this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( this.calculatedFieldsService, - this.entityService, - this.dialogService, this.translate, this.dialog, this.entityIdValue, this.store, - this.viewContainerRef, - this.overlay, - this.cd, - this.utilsService, this.durationLeft, this.popoverService, - this.destroyRef + this.destroyRef, ); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html new file mode 100644 index 0000000000..99e68cf059 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -0,0 +1,106 @@ + +
+
+
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'calculated-fields.datasource' | translate }}
+
{{ 'common.type' | translate }}
+
{{ 'entity.key' | translate }}
+
+
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + + @if (group.get('refEntityId')?.get('id').value) { + + + + + {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} + + + + + + } @else { + + + + {{ (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant + ? 'calculated-fields.argument-current-tenant' + : 'calculated-fields.argument-current') | translate }} + + + + } + + + + + {{ ArgumentTypeTranslations.get(group.get('refEntityKey').get('type').value) | translate }} + + + + + +
+ {{group.get('refEntityKey').get('key').value}} +
+ + + +
+ + +
+
+ } @empty { + {{ 'calculated-fields.no-arguments' | translate }} + } +
+ @if (errorText) { + + } +
+
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss new file mode 100644 index 0000000000..9507d9f012 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + .inline-entity-autocomplete { + .mat-mdc-form-field-infix { + padding-top: 8px; + padding-bottom: 8px; + min-height: 40px; + width: auto; + .mdc-text-field__input, .mat-mdc-select { + font-weight: 400; + line-height: 20px; + } + } + a { + font-size: 14px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts new file mode 100644 index 0000000000..0ff6ecdf7f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -0,0 +1,213 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectorRef, + Component, + DestroyRef, + effect, + forwardRef, + input, + Input, + Renderer2, + ViewContainerRef, +} from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { + ArgumentEntityType, + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgument, + CalculatedFieldArgumentValue, + CalculatedFieldType, +} from '@shared/models/calculated-field.models'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { isDefinedAndNotNull } from '@core/utils'; +import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; + +@Component({ + selector: 'tb-calculated-field-arguments-table', + templateUrl: './calculated-field-arguments-table.component.html', + styleUrls: [`calculated-field-arguments-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldArgumentsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldArgumentsTableComponent), + multi: true + } + ], +}) +export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator { + + @Input() entityId: EntityId; + @Input() tenantId: string; + + calculatedFieldType = input() + + errorText = ''; + argumentsFormArray = this.fb.array([]); + keysPopupClosed = true; + + readonly entityTypeTranslations = entityTypeTranslations; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly EntityType = EntityType; + readonly ArgumentEntityType = ArgumentEntityType; + + private onChange: (argumentsObj: Record) => void = () => {}; + + constructor( + private fb: FormBuilder, + private popoverService: TbPopoverService, + private viewContainerRef: ViewContainerRef, + private destroyRef: DestroyRef, + private cd: ChangeDetectorRef, + private renderer: Renderer2 + ) { + this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { + this.onChange(this.getArgumentsObject()); + }); + effect(() => this.calculatedFieldType() && this.argumentsFormArray.updateValueAndValidity()); + } + + registerOnChange(fn: (argumentsObj: Record) => void): void { + this.onChange = fn; + } + + registerOnTouched(_): void {} + + validate(): ValidationErrors | null { + if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE + && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { + this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; + } else if (!this.argumentsFormArray.controls.length) { + this.errorText = 'calculated-fields.hint.arguments-empty'; + } else { + this.errorText = ''; + } + return this.errorText ? { argumentsFormArray: false } : null; + } + + private getArgumentsObject(): Record { + return this.argumentsFormArray.controls.reduce((acc, control) => { + const rawValue = control.getRawValue(); + const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; + acc[argumentName] = argument; + return acc; + }, {} as Record); + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + Object.keys(argumentsObj).forEach(key => { + this.argumentsFormArray.push(this.fb.group({ + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + ...argumentsObj[key], + ...(argumentsObj[key].refEntityId ? { + refEntityId: this.fb.group({ + entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], + id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], + }), + } : {}), + refEntityKey: this.fb.group({ + type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], + key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], + }), + }) as AbstractControl); + }); + } + + + manageArgument($event: Event, matButton: MatButton, index?: number): void { + $event?.stopPropagation(); + const trigger = matButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const ctx = { + index, + argument: this.argumentsFormArray.at(index)?.getRawValue() ?? {}, + entityId: this.entityId, + calculatedFieldType: this.calculatedFieldType(), + buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', + tenantId: this.tenantId, + }; + this.keysPopupClosed = false; + const argumentsPanelPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'leftBottom', false, null, + ctx, + {}, + {}, {}, true); + argumentsPanelPopover.tbComponentRef.instance.popover = argumentsPanelPopover; + argumentsPanelPopover.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { + argumentsPanelPopover.hide(); + const formGroup = this.getArgumentFormGroup(value); + if (isDefinedAndNotNull(index)) { + this.argumentsFormArray.setControl(index, formGroup); + } else { + this.argumentsFormArray.push(formGroup); + } + this.argumentsFormArray.markAsDirty(); + this.cd.markForCheck(); + }); + argumentsPanelPopover.tbHideStart.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.keysPopupClosed = true; + }); + } + } + + getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { + return this.fb.group({ + ...value, + argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + ...(value.refEntityId ? { + refEntityId: this.fb.group({ + entityType: [{ value: value.refEntityId.entityType, disabled: true }], + id: [{ value: value.refEntityId.id , disabled: true }], + }), + } : {}), + refEntityKey: this.fb.group({ + type: [{ value: value.refEntityKey.type, disabled: true }], + key: [{ value: value.refEntityKey.key, disabled: true }], + }), + }) + } + + onDelete(index: number): void { + this.argumentsFormArray.removeAt(index); + this.argumentsFormArray.markAsDirty(); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html new file mode 100644 index 0000000000..7d1373bebb --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -0,0 +1,169 @@ + +
+ +

{{ 'entity.type-calculated-field' | translate}}

+ +
+ + +
+
+
+
{{ 'common.general' | translate }}
+
+ + {{ 'entity-field.title' | translate }} + + @if (fieldFormGroup.get('name').errors && fieldFormGroup.get('name').touched) { + + @if (fieldFormGroup.get('name').hasError('required')) { + {{ 'common.hint.name-required' | translate }} + } @else if (fieldFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.name-pattern' | translate }} + } @else if (fieldFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.name-max-length' | translate }} + } + + } + + +
+ + {{ 'common.type' | translate }} + + @for (type of fieldTypes; track type) { + {{ CalculatedFieldTypeTranslations.get(type) | translate}} + } + + +
+ +
+
{{ 'calculated-fields.arguments' | translate }}
+ +
+
+
{{ 'calculated-fields.expression' | translate }}*
+ @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { + + + @if (configFormGroup.get('expressionSIMPLE').errors && configFormGroup.get('expressionSIMPLE').touched) { + + @if (configFormGroup.get('expressionSIMPLE').hasError('required')) { + {{ 'calculated-fields.hint.expression-required' | translate }} + } @else if (configFormGroup.get('expressionSIMPLE').hasError('pattern')) { + {{ 'calculated-fields.hint.expression-invalid' | translate }} + } @else if (configFormGroup.get('expressionSIMPLE').hasError('maxLength')) { + {{ 'calculated-fields.hint.expression-max-length' | translate }} + } + + } + + } @else { + + } +
+
+
{{ 'calculated-fields.output' | translate }}
+
+ + {{ 'calculated-fields.output-type' | translate }} + + @for (type of outputTypes; track type) { + {{ OutputTypeTranslations.get(type) | translate}} + } + + + @if (outputFormGroup.get('type').value === OutputType.Attribute) { + + {{ 'calculated-fields.output-type' | translate }} + + + {{ 'calculated-fields.server-attributes' | translate }} + + @if (data.entityId.entityType === EntityType.DEVICE) { + + {{ 'calculated-fields.shared-attributes' | translate }} + + } + + + } +
+ @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { + + + {{ (outputFormGroup.get('type').value === OutputType.Timeseries + ? 'calculated-fields.timeseries-key' + : 'calculated-fields.attribute-key') + | translate }} + + + @if (outputFormGroup.get('name').errors && outputFormGroup.get('name').touched) { + + @if (outputFormGroup.get('name').hasError('required')) { + {{ 'common.hint.key-required' | translate }} + } @else if (outputFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.key-pattern' | translate }} + } @else if (outputFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.key-max-length' | translate }} + } + + } + + } +
+ +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts new file mode 100644 index 0000000000..99bf62ad46 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -0,0 +1,122 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormGroup, UntypedFormBuilder, Validators } from '@angular/forms'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { helpBaseUrl } from '@shared/models/constants'; +import { + CalculatedField, + CalculatedFieldConfiguration, + CalculatedFieldDialogData, + CalculatedFieldType, + CalculatedFieldTypeTranslations, + OutputType, + OutputTypeTranslations +} from '@shared/models/calculated-field.models'; +import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { EntityType } from '@shared/models/entity-type.models'; +import { map } from 'rxjs/operators'; +import { isObject } from '@core/utils'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; + +@Component({ + selector: 'tb-calculated-field-dialog', + templateUrl: './calculated-field-dialog.component.html', +}) +export class CalculatedFieldDialogComponent extends DialogComponent { + + fieldFormGroup = this.fb.group({ + name: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + type: [CalculatedFieldType.SIMPLE, [Validators.required]], + debugSettings: [], + configuration: this.fb.group({ + arguments: [{}], + expressionSIMPLE: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + expressionSCRIPT: [], + output: this.fb.group({ + name: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + type: [OutputType.Timeseries] + }), + }), + }); + + functionArgs$ = this.fieldFormGroup.get('configuration').valueChanges + .pipe( + map(configuration => isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) + ); + + readonly OutputTypeTranslations = OutputTypeTranslations; + readonly OutputType = OutputType; + readonly AttributeScope = AttributeScope; + readonly EntityType = EntityType; + readonly CalculatedFieldType = CalculatedFieldType; + readonly ScriptLanguage = ScriptLanguage; + readonly helpLink = `${helpBaseUrl}/[TODO: ADD VALID LINK!!!]`; + readonly fieldTypes = Object.values(CalculatedFieldType) as CalculatedFieldType[]; + readonly outputTypes = Object.values(OutputType) as OutputType[]; + readonly CalculatedFieldTypeTranslations = CalculatedFieldTypeTranslations; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDialogData, + public dialogRef: MatDialogRef, + public fb: UntypedFormBuilder) { + super(store, router, dialogRef); + this.applyDialogData(); + this.outputFormGroup.get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleScopeByOutputType(type)); + this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); + } + + get configFormGroup(): FormGroup { + return this.fieldFormGroup.get('configuration') as FormGroup; + } + + get outputFormGroup(): FormGroup { + return this.fieldFormGroup.get('configuration').get('output') as FormGroup; + } + + cancel(): void { + this.dialogRef.close(null); + } + + add(): void { + if (this.fieldFormGroup.valid) { + const { configuration, type, ...rest } = this.fieldFormGroup.value; + const { expressionSIMPLE, expressionSCRIPT, ...restConfig } = configuration; + this.dialogRef.close({ configuration: { ...restConfig, type, expression: configuration['expression'+type] }, ...rest, type }); + } + } + + private applyDialogData(): void { + const { configuration = {}, type = CalculatedFieldType.SIMPLE, ...value } = this.data.value; + const { expression, ...restConfig } = configuration as CalculatedFieldConfiguration; + this.fieldFormGroup.patchValue({ configuration: { ...restConfig, ['expression'+type]: expression }, ...value }); + } + + private toggleScopeByOutputType(type: OutputType): void { + this.outputFormGroup.get('scope')[type === OutputType.Attribute? 'enable' : 'disable']({emitEvent: false}); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html new file mode 100644 index 0000000000..26a347aaa2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -0,0 +1,198 @@ + +
+
+
{{ 'calculated-fields.argument-settings' | translate }}
+
+
+
{{ 'calculated-fields.argument-name' | translate }}
+
+ + + @if (argumentFormGroup.get('argumentName').hasError('required') && argumentFormGroup.get('argumentName').touched) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('pattern') && argumentFormGroup.get('argumentName').touched) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('maxlength') && argumentFormGroup.get('argumentName').touched) { + + warning + + } + +
+
+ +
+
{{ 'entity.entity-type' | translate }}
+ + + @for (type of argumentEntityTypes; track type) { + {{ ArgumentEntityTypeTranslations.get(type) | translate }} + } + + +
+ @if (entityType === ArgumentEntityType.Device || entityType === ArgumentEntityType.Asset) { +
+
{{ 'calculated-fields.device-name' | translate }}
+ +
+ } @else if (entityType === ArgumentEntityType.Customer) { +
+
{{ 'calculated-fields.customer-name' | translate }}
+ +
+ } + + +
+
{{ 'calculated-fields.argument-type' | translate }}
+ + + @for (type of argumentTypes; track type) { + {{ ArgumentTypeTranslations.get(type) | translate }} + } + + @if (refEntityKeyFormGroup.get('type').hasError('required') && refEntityKeyFormGroup.get('type').touched) { + + warning + + } + +
+ @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { +
+
{{ 'calculated-fields.timeseries-key' | translate }}
+ +
+ } @else { +
+
{{ 'calculated-fields.attribute-scope' | translate }}
+ + + + {{ 'calculated-fields.server-attributes' | translate }} + + @if ((keyEntityType$ | async) === EntityType.DEVICE) { + + {{ 'calculated-fields.client-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + } + + +
+
+
{{ 'calculated-fields.attribute-key' | translate }}
+ +
+ } + + @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) { +
+
{{ 'calculated-fields.default-value' | translate }}
+
+ + + +
+
+ } @else { +
+
{{ 'calculated-fields.time-window' | translate }}
+
+ + + {{ 'common.suffix.ms' | translate }} + +
+
+
+
{{ 'calculated-fields.limit' | translate }}
+
+ + + +
+
+ } +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..a784909b92 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .mat-mdc-form-field { + width: 100%; + } +} + +:host ::ng-deep { + .entity-autocomplete { + .mat-mdc-form-field { + width: 100%; + } + } +} + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts new file mode 100644 index 0000000000..2ba38beee5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -0,0 +1,201 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, ElementRef, Input, OnInit, output, ViewChild } from '@angular/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { + ArgumentEntityType, + ArgumentEntityTypeTranslations, + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgumentValue, + CalculatedFieldType +} from '@shared/models/calculated-field.models'; +import { debounceTime, delay, distinctUntilChanged, filter, map, startWith } from 'rxjs/operators'; +import { EntityType } from '@shared/models/entity-type.models'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { DatasourceType } from '@shared/models/widget.models'; +import { EntityId } from '@shared/models/id/entity-id'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { AliasFilterType } from '@shared/models/alias.models'; +import { merge } from 'rxjs'; + +@Component({ + selector: 'tb-calculated-field-argument-panel', + templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] +}) +export class CalculatedFieldArgumentPanelComponent extends PageComponent implements OnInit { + + @Input() popover: TbPopoverComponent; + @Input() buttonTitle: string; + @Input() index: number; + @Input() argument: CalculatedFieldArgumentValue; + @Input() entityId: EntityId; + @Input() tenantId: string; + @Input() calculatedFieldType: CalculatedFieldType; + + @ViewChild('timeseriesInput') timeseriesInput: ElementRef; + + argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); + + argumentFormGroup = this.fb.group({ + argumentName: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + refEntityId: this.fb.group({ + entityType: [ArgumentEntityType.Current], + id: [''] + }), + refEntityKey: this.fb.group({ + type: [ArgumentType.LatestTelemetry, [Validators.required]], + key: [''], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + }), + defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], + limit: [null], + timeWindow: [null], + }); + + argumentTypes: ArgumentType[]; + entityFilter: EntityFilter; + keyEntityType$ = this.refEntityIdFormGroup.get('entityType').valueChanges + .pipe( + startWith(this.refEntityIdFormGroup.get('entityType').value), + map(type => type === ArgumentEntityType.Current ? this.entityId.entityType : type) + ); + + readonly argumentEntityTypes = Object.values(ArgumentEntityType) as ArgumentEntityType[]; + readonly ArgumentEntityTypeTranslations = ArgumentEntityTypeTranslations; + readonly ArgumentType = ArgumentType; + readonly DataKeyType = DataKeyType; + readonly EntityType = EntityType; + readonly datasourceType = DatasourceType; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly AttributeScope = AttributeScope; + + constructor( + private fb: FormBuilder, + ) { + super(); + + this.observeEntityFilterChanges(); + this.observeEntityTypeChanges() + this.observeEntityKeyChanges(); + } + + get entityType(): ArgumentEntityType { + return this.argumentFormGroup.get('refEntityId').get('entityType').value; + } + + get refEntityIdFormGroup(): FormGroup { + return this.argumentFormGroup.get('refEntityId') as FormGroup; + } + + get refEntityKeyFormGroup(): FormGroup { + return this.argumentFormGroup.get('refEntityKey') as FormGroup; + } + + ngOnInit(): void { + this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); + this.updateEntityFilter(this.argument.refEntityId?.entityType, true); + this.toggleByEntityKeyType(this.argument.refEntityKey?.type); + this.setInitialEntityKeyType(); + + this.argumentTypes = Object.values(ArgumentType) + .filter(type => type !== ArgumentType.Rolling || this.calculatedFieldType === CalculatedFieldType.SCRIPT); + } + + saveArgument(): void { + this.argumentsDataApplied.emit({ value: this.argumentFormGroup.value as CalculatedFieldArgumentValue, index: this.index }); + } + + cancel(): void { + this.popover.hide(); + } + + private toggleByEntityKeyType(type: ArgumentType): void { + const isAttribute = type === ArgumentType.Attribute; + const isRolling = type === ArgumentType.Rolling; + this.argumentFormGroup.get('refEntityKey').get('scope')[isAttribute? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('limit')[isRolling? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('timeWindow')[isRolling? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('defaultValue')[isRolling? 'disable' : 'enable']({ emitEvent: false }); + } + + private updateEntityFilter(entityType: ArgumentEntityType, onInit = false): void { + let entityId: EntityId; + switch (entityType) { + case ArgumentEntityType.Current: + entityId = this.entityId + break; + case ArgumentEntityType.Tenant: + entityId = { + id: this.tenantId, + entityType: EntityType.TENANT + }; + break; + default: + entityId = this.argumentFormGroup.get('refEntityId').value as any; + } + if (onInit) { + this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); + } + this.entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: entityId, + }; + } + + private observeEntityFilterChanges(): void { + merge( + this.argumentFormGroup.get('refEntityId').get('entityType').valueChanges, + this.argumentFormGroup.get('refEntityId').get('id').valueChanges.pipe(filter(Boolean)), + this.argumentFormGroup.get('refEntityKey').get('scope').valueChanges, + ) + .pipe(debounceTime(300), delay(50), takeUntilDestroyed()) + .subscribe(() => this.updateEntityFilter(this.entityType)); + } + + private observeEntityTypeChanges(): void { + this.argumentFormGroup.get('refEntityId').get('entityType').valueChanges + .pipe(distinctUntilChanged(), takeUntilDestroyed()) + .subscribe(type => { + this.argumentFormGroup.get('refEntityId').get('id').setValue(''); + this.argumentFormGroup.get('refEntityId') + .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); + }); + } + + private observeEntityKeyChanges(): void { + this.argumentFormGroup.get('refEntityKey').get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleByEntityKeyType(type)); + } + + private setInitialEntityKeyType(): void { + if (this.calculatedFieldType === CalculatedFieldType.SIMPLE && this.argument.refEntityKey?.type === ArgumentType.Rolling) { + const typeControl = this.argumentFormGroup.get('refEntityKey').get('type'); + typeControl.setValue(null); + typeControl.markAsTouched(); + typeControl.updateValueAndValidity(); + } + } + + protected readonly ArgumentEntityType = ArgumentEntityType; +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts new file mode 100644 index 0000000000..c3d1ede02e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -0,0 +1,18 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export * from './dialog/calculated-field-dialog.component'; +export * from './arguments-table/calculated-field-arguments-table.component'; diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 1a6b9c08e0..f60ea15407 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -185,6 +185,16 @@ import { EntityChipsComponent } from '@home/components/entity/entity-chips.compo import { DashboardViewComponent } from '@home/components/dashboard-view/dashboard-view.component'; import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { CalculatedFieldDialogComponent } from '@home/components/calculated-fields/components/dialog/calculated-field-dialog.component'; +import { + EntityDebugSettingsButtonComponent +} from '@home/components/entity/debug/entity-debug-settings-button.component'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; @NgModule({ declarations: @@ -330,6 +340,9 @@ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; EntityChipsComponent, DashboardViewComponent, CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldArgumentsTableComponent, + CalculatedFieldArgumentPanelComponent, ], imports: [ CommonModule, @@ -341,7 +354,8 @@ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; SnmpDeviceProfileTransportModule, StatesControllerModule, DeviceCredentialsModule, - DeviceProfileCommonModule + DeviceProfileCommonModule, + EntityDebugSettingsButtonComponent ], exports: [ RouterTabsComponent, @@ -468,6 +482,9 @@ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; EntityChipsComponent, DashboardViewComponent, CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldArgumentsTableComponent, + CalculatedFieldArgumentPanelComponent, ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts index 618650ac32..fbbf124629 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts @@ -19,6 +19,7 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { DeviceInfo } from '@shared/models/device.models'; import { EntityTabsComponent } from '../../components/entity/entity-tabs.component'; +import { EntityType } from '@shared/models/entity-type.models'; @Component({ selector: 'tb-device-tabs', @@ -27,6 +28,8 @@ import { EntityTabsComponent } from '../../components/entity/entity-tabs.compone }) export class DeviceTabsComponent extends EntityTabsComponent { + readonly EntityType = EntityType; + constructor(protected store: Store) { super(store); } @@ -34,5 +37,4 @@ export class DeviceTabsComponent extends EntityTabsComponent { ngOnInit() { super.ngOnInit(); } - } diff --git a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html index c194e04290..44a218b455 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html @@ -17,10 +17,11 @@ --> - {{ label | translate }} + {{ label | translate }} {{ displayEntityFn(selectEntityFormGroup.get('entity').value) }} + + warning + + } + + @for (key of filteredKeys$ | async; track key) { + + } + + diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts new file mode 100644 index 0000000000..2a33a74786 --- /dev/null +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts @@ -0,0 +1,134 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, effect, ElementRef, forwardRef, input, ViewChild, } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { map, startWith, switchMap } from 'rxjs/operators'; +import { combineLatest, of, Subject } from 'rxjs'; +import { EntityService } from '@core/http/entity.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { EntitiesKeysByQuery } from '@shared/models/entity.models'; +import { EntityFilter } from '@shared/models/query/query.models'; + +@Component({ + selector: 'tb-entity-key-autocomplete', + templateUrl: './entity-key-autocomplete.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityKeyAutocompleteComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityKeyAutocompleteComponent), + multi: true + } + ], + host: { + class: 'w-full' + } +}) +export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Validator { + + @ViewChild('keyInput') keyInput: ElementRef; + + entityFilter = input.required(); + dataKeyType = input.required(); + keyScopeType = input(); + + keyControl = this.fb.control('', [Validators.required]); + searchText = ''; + keyInputSubject = new Subject(); + + private onChange: (value: string) => void; + private cachedResult: EntitiesKeysByQuery; + + keys$ = this.keyInputSubject.asObservable() + .pipe( + switchMap(() => { + return this.cachedResult ? of(this.cachedResult) : this.entityService.findEntityKeysByQuery({ + pageLink: { page: 0, pageSize: 100 }, + entityFilter: this.entityFilter(), + }, this.dataKeyType() === DataKeyType.attribute, this.dataKeyType() === DataKeyType.timeseries, this.keyScopeType()); + }), + map(result => { + this.cachedResult = result; + switch (this.dataKeyType()) { + case DataKeyType.attribute: + return result.attribute; + case DataKeyType.timeseries: + return result.timeseries; + default: + return []; + } + }), + ); + + filteredKeys$ = combineLatest([this.keys$, this.keyControl.valueChanges.pipe(startWith(''))]) + .pipe( + map(([keys, searchText = '']) => { + this.searchText = searchText; + return searchText ? keys.filter(item => item.toLowerCase().includes(searchText.toLowerCase())) : keys; + }) + ); + + constructor( + private fb: FormBuilder, + private entityService: EntityService, + ) { + this.keyControl.valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.onChange(value)); + effect(() => { + if (this.keyScopeType() || this.entityFilter() && this.dataKeyType()) { + this.cachedResult = null; + this.searchText = ''; + } + }); + } + + clear(): void { + this.keyControl.patchValue('', {emitEvent: true}); + setTimeout(() => { + this.keyInput.nativeElement.blur(); + this.keyInput.nativeElement.focus(); + }, 0); + } + + registerOnChange(onChange: (value: string) => void): void { + this.onChange = onChange; + } + + registerOnTouched(_): void {} + + validate(): ValidationErrors | null { + return this.keyControl.valid ? null : { keyControl: false }; + } + + writeValue(value: string): void { + this.keyControl.patchValue(value, {emitEvent: false}); + } +} diff --git a/ui-ngx/src/app/shared/components/js-func.component.ts b/ui-ngx/src/app/shared/components/js-func.component.ts index 8e500bdbe9..57c15a5484 100644 --- a/ui-ngx/src/app/shared/components/js-func.component.ts +++ b/ui-ngx/src/app/shared/components/js-func.component.ts @@ -20,9 +20,11 @@ import { ElementRef, forwardRef, Input, + OnChanges, OnDestroy, OnInit, Renderer2, + SimpleChanges, ViewChild, ViewContainerRef, ViewEncapsulation @@ -67,7 +69,7 @@ import { catchError } from 'rxjs/operators'; ], encapsulation: ViewEncapsulation.None }) -export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { +export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlValueAccessor, Validator { @ViewChild('javascriptEditor', {static: true}) javascriptEditorElmRef: ElementRef; @@ -177,6 +179,13 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, private http: HttpClient) { } + ngOnChanges(changes: SimpleChanges): void { + if (changes.functionArgs) { + this.updateFunctionArgsString(); + this.updateFunctionLabel(); + } + } + ngOnInit(): void { if (this.functionTitle || this.label) { this.hideBrackets = true; @@ -184,22 +193,6 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, if (!this.resultType || this.resultType.length === 0) { this.resultType = 'nocheck'; } - if (this.functionArgs) { - this.functionArgs.forEach((functionArg) => { - if (this.functionArgsString.length > 0) { - this.functionArgsString += ', '; - } - this.functionArgsString += functionArg; - }); - } - if (this.functionTitle) { - this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; - } else if (this.label) { - this.functionLabel = this.label; - } else { - this.functionLabel = - `function ${this.functionName ? this.functionName : ''}(${this.functionArgsString})${this.hideBrackets ? '' : ' {'}`; - } const editorElement = this.javascriptEditorElmRef.nativeElement; let editorOptions: Partial = { mode: 'ace/mode/javascript', @@ -329,6 +322,25 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, ); } + private updateFunctionArgsString(): void { + this.functionArgsString = ''; + if (this.functionArgs) { + this.functionArgsString = this.functionArgs.join(', '); + } + } + + private updateFunctionLabel(): void { + if (this.functionTitle) { + this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; + } else if (this.label) { + this.functionLabel = this.label; + } else { + this.functionLabel = + `function ${this.functionName ? this.functionName : ''}(${this.functionArgsString})${this.hideBrackets ? '' : ' {'}`; + } + this.cd.markForCheck(); + } + validateOnSubmit(): Observable { if (!this.disabled) { this.cleanupJsErrors(); diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 253bc58f39..73b3ecd861 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -13,29 +13,109 @@ /// See the License for the specific language governing permissions and /// limitations under the License. /// + import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/entity.models'; import { BaseData } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; +import { EntityId } from '@shared/models/id/entity-id'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { - type: CalculatedFieldType; debugSettings?: EntityDebugSettings; externalId?: string; configuration: CalculatedFieldConfiguration; + type: CalculatedFieldType; } export enum CalculatedFieldType { SIMPLE = 'SIMPLE', - COMPLEX = 'COMPLEX', + SCRIPT = 'SCRIPT', } +export const CalculatedFieldTypeTranslations = new Map( + [ + [CalculatedFieldType.SIMPLE, 'calculated-fields.type.simple'], + [CalculatedFieldType.SCRIPT, 'calculated-fields.type.script'], + ] +) + export interface CalculatedFieldConfiguration { - type: CalculatedFieldConfigType; + type: CalculatedFieldType; expression: string; - arguments: Record; + arguments: Record; } -export enum CalculatedFieldConfigType { - SIMPLE = 'SIMPLE', - SCRIPT = 'SCRIPT', +export enum ArgumentEntityType { + Current = 'CURRENT', + Device = 'DEVICE', + Asset = 'ASSET', + Customer = 'CUSTOMER', + Tenant = 'TENANT', +} + +export const ArgumentEntityTypeTranslations = new Map( + [ + [ArgumentEntityType.Current, 'calculated-fields.argument-current'], + [ArgumentEntityType.Device, 'calculated-fields.argument-device'], + [ArgumentEntityType.Asset, 'calculated-fields.argument-asset'], + [ArgumentEntityType.Customer, 'calculated-fields.argument-customer'], + [ArgumentEntityType.Tenant, 'calculated-fields.argument-tenant'], + ] +) + +export enum ArgumentType { + Attribute = 'ATTRIBUTE', + LatestTelemetry = 'TS_LATEST', + Rolling = 'TS_ROLLING', +} + +export enum OutputType { + Attribute = 'ATTRIBUTES', + Timeseries = 'TIME_SERIES', +} + +export const OutputTypeTranslations = new Map( + [ + [OutputType.Attribute, 'calculated-fields.attribute'], + [OutputType.Timeseries, 'calculated-fields.timeseries'], + ] +) + +export const ArgumentTypeTranslations = new Map( + [ + [ArgumentType.Attribute, 'calculated-fields.attribute'], + [ArgumentType.LatestTelemetry, 'calculated-fields.latest-telemetry'], + [ArgumentType.Rolling, 'calculated-fields.rolling'], + ] +) + +export interface CalculatedFieldArgument { + refEntityKey: RefEntityKey; + defaultValue?: string; + refEntityId?: RefEntityKey; + limit?: number; + timeWindow?: number; +} + +export interface RefEntityKey { + key: string; + type: ArgumentType; + scope?: AttributeScope; +} + +export interface RefEntityKey { + entityType: ArgumentEntityType; + id: string; +} + +export interface CalculatedFieldArgumentValue extends CalculatedFieldArgument { + argumentName: string; +} + +export interface CalculatedFieldDialogData { + value: CalculatedField; + buttonTitle: string; + entityId: EntityId; + debugLimitsConfiguration: string; + tenantId: string; } diff --git a/ui-ngx/src/app/shared/models/public-api.ts b/ui-ngx/src/app/shared/models/public-api.ts index 736d885810..c48a3286cd 100644 --- a/ui-ngx/src/app/shared/models/public-api.ts +++ b/ui-ngx/src/app/shared/models/public-api.ts @@ -61,3 +61,4 @@ export * from './widgets-bundle.model'; export * from './window-message.model'; export * from './usage.models'; export * from './query/query.models'; +export * from './regex.constants'; diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts new file mode 100644 index 0000000000..55742cefd4 --- /dev/null +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -0,0 +1,17 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 5b40dadf5f..5825998434 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -224,6 +224,7 @@ import { IntervalOptionsConfigPanelComponent } from '@shared/components/time/int import { GroupingIntervalOptionsComponent } from '@shared/components/time/aggregation/grouping-interval-options.component'; import { JsFuncModulesComponent } from '@shared/components/js-func-modules.component'; import { JsFuncModuleRowComponent } from '@shared/components/js-func-module-row.component'; +import { EntityKeyAutocompleteComponent } from '@shared/components/entity/entity-key-autocomplete.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { return markedOptionsService; @@ -432,7 +433,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) ImageGalleryDialogComponent, WidgetButtonComponent, HexInputComponent, - ScadaSymbolInputComponent + ScadaSymbolInputComponent, + EntityKeyAutocompleteComponent, ], imports: [ CommonModule, @@ -694,7 +696,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) EmbedImageDialogComponent, ImageGalleryDialogComponent, WidgetButtonComponent, - ScadaSymbolInputComponent + ScadaSymbolInputComponent, + EntityKeyAutocompleteComponent, ] }) export class SharedModule { } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index beb6734451..be31b4f70b 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -996,6 +996,7 @@ "failures": "Failures", "entity": "entity", "rule-node": "rule node", + "calculated-field": "calculated field", "hint": { "main": "All node debug messages rate limited with:", "main-limited": "All {{entity}} debug messages will be rate-limited, with a maximum of {{msg}} messages allowed per {{time}}.", @@ -1007,7 +1008,56 @@ "expression": "Expression", "no-found": "No calculated fields found", "list": "{ count, plural, =1 {One calculated field} other {List of # calculated fields} }", - "selected-fields": "{ count, plural, =1 {1 calculated field} other {# calculated fields} } selected" + "selected-fields": "{ count, plural, =1 {1 calculated field} other {# calculated fields} } selected", + "type": { + "simple": "Simple", + "script": "Script" + }, + "arguments": "Arguments", + "argument-name": "Argument name", + "datasource": "Datasource", + "add-argument": "Add argument", + "no-arguments": "No arguments configured", + "argument-settings": "Argument settings", + "argument-current": "Current entity", + "argument-current-tenant": "Current tenant", + "argument-device": "Device", + "argument-asset": "Asset", + "argument-customer": "Customer", + "argument-tenant": "Current tenant", + "argument-type": "Argument type", + "attribute": "Attribute", + "timeseries-key": "Time series key", + "device-name": "Device name", + "latest-telemetry": "Latest telemetry", + "rolling": "Rolling", + "attribute-scope": "Attribute scope", + "server-attributes": "Server attributes", + "client-attributes": "Client attributes", + "shared-attributes": "Shared attributes", + "attribute-key": "Attribute key", + "default-value": "Default value", + "limit": "Limit", + "time-window": "Time window", + "customer-name": "Customer name", + "timeseries": "Time series", + "output": "Output", + "output-type": "Output type", + "delete-title": "Are you sure you want to delete the calculated field '{{title}}'?", + "delete-text": "Be careful, after the confirmation the calculated field and all related data will become unrecoverable.", + "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", + "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", + "hint": { + "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with rolling type.", + "arguments-empty": "Arguments should not be empty.", + "expression-required": "Expression is required.", + "expression-invalid": "Expression is invalid", + "expression-max-length": "Expression length should be less than 255 characters.", + "argument-name-required": "Argument name is required.", + "argument-name-pattern": "Argument name is invalid.", + "argument-name-max-length": "Argument name should be less than 256 characters.", + "argument-type-required": "Argument type is required." + } }, "confirm-on-exit": { "message": "You have unsaved changes. Are you sure you want to leave this page?", @@ -1035,6 +1085,7 @@ "common": { "name": "Name", "type": "Type", + "general": "General", "username": "Username", "password": "Password", "enter-username": "Enter username", @@ -1047,7 +1098,19 @@ "open-details-page": "Open details page", "not-found": "Not found", "documentation": "Documentation", - "time-left": "{{time}} left" + "time-left": "{{time}} left", + "suffix": { + "s": "s", + "ms": "ms" + }, + "hint": { + "name-required": "Name is required.", + "name-pattern": "Name is invalid.", + "name-max-length": "Name should be less than 256 characters.", + "key-required": "Key is required.", + "key-pattern": "Key is invalid.", + "key-max-length": "Key should be less than 256 characters." + } }, "content-type": { "json": "Json", From c1c9fb4f5aa9ee4fa5c369ba51436b194d908b1f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Jan 2025 11:29:38 +0200 Subject: [PATCH 145/416] implemented calculated field debug events persistence --- .../server/actors/ActorSystemContext.java | 52 +++++++++++-------- ...CalculatedFieldEntityMessageProcessor.java | 42 ++++++++++----- ...alculatedFieldManagerMessageProcessor.java | 20 +++---- ...efaultCalculatedFieldExecutionService.java | 15 ++++-- .../ctx/state/BaseCalculatedFieldState.java | 19 ++++--- .../cf/ctx/state/CalculatedFieldCtx.java | 42 +++++++++++++-- .../cf/ctx/state/CalculatedFieldState.java | 4 ++ .../ctx/state/ScriptCalculatedFieldState.java | 6 +++ .../ctx/state/SimpleCalculatedFieldState.java | 15 ++---- .../cf/DefaultTbCalculatedFieldService.java | 10 ++++ common/proto/src/main/proto/queue.proto | 3 ++ ...efaultNativeCalculatedFieldRepository.java | 3 ++ .../engine/api/AttributesSaveRequest.java | 18 ++++++- .../engine/api/TimeseriesSaveRequest.java | 18 ++++++- .../engine/telemetry/TbMsgAttributesNode.java | 2 + .../engine/telemetry/TbMsgTimeseriesNode.java | 2 + 16 files changed, 194 insertions(+), 77 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 09488bfe4e..daa6744fd9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -50,6 +50,7 @@ import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; @@ -105,6 +106,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; @@ -129,6 +131,7 @@ import org.thingsboard.server.service.transport.TbCoreToTransportService; import java.io.PrintWriter; import java.io.StringWriter; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; @@ -744,29 +747,34 @@ public class ActorSystemContext { } } - public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, TbMsg tbMsg, Throwable error) { - if (checkLimits(tenantId, tbMsg, error)) { - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder event = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(entityId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(tbMsg.getOriginator()) - .msgId(tbMsg.getId()) - .msgType(tbMsg.getType()) - .arguments(JacksonUtil.toString(arguments)) - .result(tbMsg.getData()); - - if (error != null) { - event.error(toString(error)); - } - - ListenableFuture future = eventService.saveAsync(event.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString(arguments)); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(toString(error)); + } + + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index cea3cce791..6426be8a3c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -18,19 +18,18 @@ package org.thingsboard.server.actors.calculatedField; import com.google.common.util.concurrent.ListenableFuture; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; +import org.thingsboard.common.util.DebugModeUtil; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -53,9 +52,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; /** @@ -111,9 +108,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM callback.onSuccess(CALLBACKS_PER_CF); } else { if (proto.getTsDataCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList())); + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getAttrDataCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList())); + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else { callback.onSuccess(CALLBACKS_PER_CF); } @@ -136,27 +133,30 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { - processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList())); + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } @SneakyThrows private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { - processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList())); + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } @SneakyThrows private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, - Map newArgValues) { + Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) { if (newArgValues.isEmpty()) { callback.onSuccess(CALLBACKS_PER_CF); } CalculatedFieldState state = getOrInitState(ctx); if (state.updateState(newArgValues)) { - if (state.isReady()) { + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); cfIdList = new ArrayList<>(cfIdList); cfIdList.add(ctx.getCfId()); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); + } } else { callback.onSuccess(); // State was updated but no calculation performed; } @@ -183,13 +183,27 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return state; } + private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { + if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { + return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); + } + return null; + } + + private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTbMsgType().isEmpty()) { + return TbMsgType.valueOf(proto.getTbMsgType()); + } + return null; + } + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { return mapToArguments(ctx.getMainEntityArguments(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, data); @@ -221,7 +235,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, scope, attrDataList); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4bce7cb322..603ab96a8e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,16 +16,14 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; +import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -38,16 +36,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.queue.discovery.HashPartitionService; -import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -101,6 +90,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onFieldInitMsg(CalculatedFieldInitMsg msg) { var cf = msg.getCf(); var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + try { + cfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(cf)) { + systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 9c8aafc47d..8f948aa265 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -833,7 +833,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); List entries = request.getEntries(); List versions = result.getVersions(); for (int i = 0; i < entries.size(); i++) { @@ -849,7 +849,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); List entries = request.getEntries(); for (int i = 0; i < entries.size(); i++) { @@ -862,7 +862,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return msg.build(); } - private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds) { + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); @@ -878,6 +878,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + if (tbMsgId != null) { + telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); + telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); + } + + if (tbMsgType != null) { + telemetryMsg.setTbMsgType(tbMsgType.name()); + } + return telemetryMsg; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index d623043cee..f5f681b505 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,19 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.NoArgsConstructor; + import java.util.HashMap; import java.util.List; import java.util.Map; +@NoArgsConstructor public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected List requiredArguments; protected Map arguments; - public BaseCalculatedFieldState() { - this.arguments = new HashMap<>(); - } - public BaseCalculatedFieldState(List requiredArguments) { this.requiredArguments = requiredArguments; this.arguments = new HashMap<>(); @@ -35,7 +34,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @Override public Map getArguments() { - return this.arguments; + return arguments; + } + + @Override + public List getRequiredArguments() { + return requiredArguments; } @Override @@ -53,7 +57,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (existingEntry == null) { validateNewEntry(newEntry); - arguments.put(key, newEntry.copy()); + arguments.put(key, newEntry); stateUpdated = true; } else { stateUpdated = existingEntry.updateEntry(newEntry); @@ -70,7 +74,6 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { !arguments.containsValue(TsRollingArgumentEntry.EMPTY); } - protected void validateNewEntry(ArgumentEntry newEntry) { - } + protected abstract void validateNewEntry(ArgumentEntry newEntry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 42536188ce..33421a7d10 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -17,6 +17,8 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; +import org.mvel2.MVEL; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; @@ -33,7 +35,6 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; @@ -45,6 +46,8 @@ import java.util.stream.Collectors; @Data public class CalculatedFieldCtx { + private CalculatedField calculatedField; + private CalculatedFieldId cfId; private TenantId tenantId; private EntityId entityId; @@ -61,7 +64,11 @@ public class CalculatedFieldCtx { private CalculatedFieldScriptEngine calculatedFieldScriptEngine; private ThreadLocal customExpression; + private boolean initialized; + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + this.calculatedField = calculatedField; + this.cfId = calculatedField.getId(); this.tenantId = calculatedField.getTenantId(); this.entityId = calculatedField.getEntityId(); @@ -88,10 +95,28 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { - this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + } + + public void init() { + if (CalculatedFieldType.SCRIPT.equals(cfType)) { + try { + this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + initialized = true; + } catch (Exception e) { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); + } } else { - this.customExpression = new ThreadLocal<>(); + if (isValidExpression(expression)) { + this.customExpression = ThreadLocal.withInitial(() -> + new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(this.arguments.keySet()) + .build() + ); + initialized = true; + } else { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax."); + } } } @@ -108,6 +133,15 @@ public class CalculatedFieldCtx { ); } + private boolean isValidExpression(String expression) { + try { + MVEL.compileExpression(expression); + return true; + } catch (Exception e) { + return false; + } + } + public boolean matches(List values, AttributeScope scope) { return matchesAttributes(mainEntityArguments, values, scope); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 4b7918cc03..173d299da6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import java.util.List; import java.util.Map; @JsonTypeInfo( @@ -40,9 +41,12 @@ public interface CalculatedFieldState { Map getArguments(); + List getRequiredArguments(); + boolean updateState(Map argumentValues); ListenableFuture performCalculation(CalculatedFieldCtx ctx); + @JsonIgnore boolean isReady(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 290fd95370..a3c7efb388 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -31,6 +32,7 @@ import java.util.TreeMap; @Data @Slf4j +@NoArgsConstructor public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { public ScriptCalculatedFieldState(List requiredArguments) { @@ -42,6 +44,10 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { return CalculatedFieldType.SCRIPT; } + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + } + @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { arguments.forEach((key, argumentEntry) -> { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 59b3009bb4..8b8fe6e8c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -18,8 +18,7 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; -import net.objecthunter.exp4j.Expression; -import net.objecthunter.exp4j.ExpressionBuilder; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -28,6 +27,7 @@ import java.util.List; import java.util.Map; @Data +@NoArgsConstructor public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { public SimpleCalculatedFieldState(List requiredArguments) { @@ -48,16 +48,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - String expression = ctx.getExpression(); - ThreadLocal customExpression = ctx.getCustomExpression(); - var expr = customExpression.get(); - if (expr == null) { - expr = new ExpressionBuilder(expression) - .implicitMultiplication(true) - .variables(this.arguments.keySet()) - .build(); - customExpression.set(expr); - } + var expr = ctx.getCustomExpression().get(); for (Map.Entry entry : this.arguments.entrySet()) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index c8c589691b..184092720c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -58,6 +58,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = calculatedField.getTenantId(); try { + if (ActionType.UPDATED.equals(actionType)) { + CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); + checkForEntityChange(existingCf, calculatedField); + } checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId()); checkArgumentSize(calculatedField.getConfiguration()); @@ -98,6 +102,12 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void checkForEntityChange(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (!oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId())) { + throw new IllegalArgumentException("Changing the calculated field target entity after initialization is prohibited."); + } + } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 69b912120e..53045fc147 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -799,6 +799,9 @@ message CalculatedFieldTelemetryMsgProto { repeated TsKvProto tsData = 9; AttributeScopeProto scope = 10; repeated AttributeValueProto attrData = 11; + int64 tbMsgIdMSB = 12; + int64 tbMsgIdLSB = 13; + string tbMsgType = 14; } message CalculatedFieldLinkedTelemetryMsgProto { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index fcbf4d4dd0..daca6d5056 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -78,6 +79,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF int configurationVersion = (int) row.get("configuration_version"); JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); long version = row.get("version") != null ? (long) row.get("version") : 0; + String debugSettings = (String) row.get("debug_settings"); Object externalIdObj = row.get("external_id"); CalculatedField calculatedField = new CalculatedField(); @@ -90,6 +92,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); + calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); return calculatedField; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index 406b2d0d5c..74f3468f49 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -28,8 +28,10 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @ToString @@ -42,6 +44,8 @@ public class AttributesSaveRequest { private final List entries; private final boolean notifyDevice; private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; public static Builder builder() { @@ -56,6 +60,8 @@ public class AttributesSaveRequest { private List entries; private boolean notifyDevice = true; private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback callback; Builder() {} @@ -108,6 +114,16 @@ public class AttributesSaveRequest { return this; } + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -128,7 +144,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, previousCalculatedFieldIds, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 95eb788e5f..957b0cf108 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -27,8 +27,10 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @AllArgsConstructor(access = AccessLevel.PRIVATE) @@ -42,6 +44,8 @@ public class TimeseriesSaveRequest { private final boolean saveLatest; private final boolean onlyLatest; private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; public static Builder builder() { @@ -59,6 +63,8 @@ public class TimeseriesSaveRequest { private boolean saveLatest = true; private boolean onlyLatest; private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; Builder() {} @@ -111,6 +117,16 @@ public class TimeseriesSaveRequest { return this; } + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -131,7 +147,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, previousCalculatedFieldIds, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index ae2fce6575..925d70daa1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -126,6 +126,8 @@ public class TbMsgAttributesNode implements TbNode { .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .callback(callback) .build()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 89f7844313..3287cc825f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -113,6 +113,8 @@ public class TbMsgTimeseriesNode implements TbNode { .ttl(ttl) .saveLatest(!config.isSkipLatestPersistence()) .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } From c01024f6da8b66b2a78adcde6884d40bfd15cabc Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Jan 2025 12:11:09 +0200 Subject: [PATCH 146/416] changed endpoint --- .../server/controller/CalculatedFieldController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 124dd8d710..96a97aeeff 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -94,11 +94,11 @@ public class CalculatedFieldController extends BaseController { return calculatedField; } - @ApiOperation(value = "Get Calculated Fields (getCalculatedFieldsByEntityId)", + @ApiOperation(value = "Get Calculated Fields by Entity Id (getCalculatedFieldsByEntityId)", notes = "Fetch the Calculated Fields based on the provided Entity Id." ) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/{entityType}/{entityId}/calculatedField", params = {"pageSize", "page"}, method = RequestMethod.GET) + @RequestMapping(value = "/{entityType}/{entityId}/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) @ResponseBody public PageData getCalculatedFieldsByEntityId( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, From a82d3690f3e282bb3ef18aec90bda15f69a387ff Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 12:30:36 +0200 Subject: [PATCH 147/416] Changed getAll endpoint and refactoring --- .../core/http/calculated-fields.service.ts | 5 +- .../calculated-fields-table-config.ts | 16 +- ...lated-field-arguments-table.component.html | 138 +++++++++--------- ...lated-field-arguments-table.component.scss | 1 + ...culated-field-arguments-table.component.ts | 101 +++++++------ .../calculated-field-dialog.component.html | 4 +- .../calculated-field-dialog.component.ts | 4 +- ...ulated-field-argument-panel.component.html | 53 +++---- ...lculated-field-argument-panel.component.ts | 9 +- .../components/public-api.ts | 1 + .../pages/device/device-tabs.component.html | 3 +- .../pages/device/device-tabs.component.ts | 4 +- .../entity-key-autocomplete.component.ts | 9 +- .../shared/models/calculated-field.models.ts | 1 + 14 files changed, 179 insertions(+), 170 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index dca0c9a3c7..10ad366e5b 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -21,6 +21,7 @@ import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; +import { EntityId } from '@shared/models/id/entity-id'; @Injectable({ providedIn: 'root' @@ -43,8 +44,8 @@ export class CalculatedFieldsService { return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public getCalculatedFields(pageLink: PageLink, config?: RequestConfig): Observable> { - return this.http.get>(`/api/calculatedFields${pageLink.toQuery()}`, + public getCalculatedFields({ entityType, id }: EntityId, pageLink: PageLink, config?: RequestConfig): Observable> { + return this.http.get>(`/api/${entityType}/${id}/calculatedFields${pageLink.toQuery()}`, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 2500c92336..ae025a13f6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -40,6 +40,7 @@ import { CalculatedFieldDialogComponent } from './components/public-api'; export class CalculatedFieldsTableConfig extends EntityTableConfig { + // TODO: [Calculated Fields] remove hardcode when BE variable implemented readonly calculatedFieldsDebugPerTenantLimitsConfiguration = getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; @@ -66,9 +67,9 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); + this.entitiesFetchFunction = (pageLink: TimePageLink) => this.fetchCalculatedFields(pageLink); this.addEntity = this.addCalculatedField.bind(this); - this.deleteEntityTitle = (field) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); + this.deleteEntityTitle = (field: CalculatedField) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); this.deleteEntitiesTitle = count => this.translate.instant('calculated-fields.delete-multiple-title', {count}); this.deleteEntitiesContent = () => this.translate.instant('calculated-fields.delete-multiple-text'); @@ -102,7 +103,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { - return this.calculatedFieldsService.getCalculatedFields(pageLink); + return this.calculatedFieldsService.getCalculatedFields(this.entityId, pageLink); } onOpenDebugConfig($event: Event, { debugSettings = {}, id }: CalculatedField): void { @@ -134,10 +135,9 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField} as any)), + switchMap(calculatedField => this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField })), ) .subscribe((res) => { if (res) { @@ -148,10 +148,9 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField} as any)), + switchMap((updatedCalculatedField) => this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField })), ) .subscribe((res) => { if (res) { @@ -160,7 +159,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { return this.dialog.open(CalculatedFieldDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], @@ -172,6 +171,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig{{ 'common.type' | translate }}
{{ 'entity.key' | translate }}
-
- @for (group of argumentsFormArray.controls; track group) { -
- - - - @if (group.get('refEntityId')?.get('id').value) { - - - - - {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} - - - - - - } @else { - - - - {{ (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant - ? 'calculated-fields.argument-current-tenant' - : 'calculated-fields.argument-current') | translate }} - - - - } - +
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + + @if (group.get('refEntityId')?.get('id').value) { + - - - {{ ArgumentTypeTranslations.get(group.get('refEntityKey').get('type').value) | translate }} + + + {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} - - -
- {{group.get('refEntityKey').get('key').value}} -
- - + -
- - -
+ } @else { + + + + {{ + (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant + ? 'calculated-fields.argument-current-tenant' + : 'calculated-fields.argument-current') | translate + }} + + + + } + + + @if (group.get('refEntityKey').get('type').value; as type) { + + + {{ ArgumentTypeTranslations.get(type) | translate }} + + + } + + + +
+ {{ group.get('refEntityKey').get('key').value }} +
+ + + +
+ +
- } @empty { - {{ 'calculated-fields.no-arguments' | translate }} - } -
+
+ } @empty { + {{ 'calculated-fields.no-arguments' | translate }} + } +
@if (errorText) { } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 9507d9f012..8695ee4068 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -25,6 +25,7 @@ line-height: 20px; } } + a { font-size: 14px; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 0ff6ecdf7f..1dedc5e80e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -43,9 +43,7 @@ import { CalculatedFieldArgumentValue, CalculatedFieldType, } from '@shared/models/calculated-field.models'; -import { - CalculatedFieldArgumentPanelComponent -} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { CalculatedFieldArgumentPanelComponent } from '@home/components/calculated-fields/components/public-api'; import { MatButton } from '@angular/material/button'; import { TbPopoverService } from '@shared/components/popover.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @@ -87,7 +85,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; - private onChange: (argumentsObj: Record) => void = () => {}; + private propagateChange: (argumentsObj: Record) => void = () => {}; constructor( private fb: FormBuilder, @@ -98,59 +96,27 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private renderer: Renderer2 ) { this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { - this.onChange(this.getArgumentsObject()); + this.propagateChange(this.getArgumentsObject()); }); effect(() => this.calculatedFieldType() && this.argumentsFormArray.updateValueAndValidity()); } registerOnChange(fn: (argumentsObj: Record) => void): void { - this.onChange = fn; + this.propagateChange = fn; } registerOnTouched(_): void {} validate(): ValidationErrors | null { - if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE - && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { - this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; - } else if (!this.argumentsFormArray.controls.length) { - this.errorText = 'calculated-fields.hint.arguments-empty'; - } else { - this.errorText = ''; - } + this.updateErrorText(); return this.errorText ? { argumentsFormArray: false } : null; } - private getArgumentsObject(): Record { - return this.argumentsFormArray.controls.reduce((acc, control) => { - const rawValue = control.getRawValue(); - const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; - acc[argumentName] = argument; - return acc; - }, {} as Record); - } - - writeValue(argumentsObj: Record): void { - this.argumentsFormArray.clear(); - Object.keys(argumentsObj).forEach(key => { - this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], - ...argumentsObj[key], - ...(argumentsObj[key].refEntityId ? { - refEntityId: this.fb.group({ - entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], - id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], - }), - } : {}), - refEntityKey: this.fb.group({ - type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], - key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], - }), - }) as AbstractControl); - }); + onDelete(index: number): void { + this.argumentsFormArray.removeAt(index); + this.argumentsFormArray.markAsDirty(); } - manageArgument($event: Event, matButton: MatButton, index?: number): void { $event?.stopPropagation(); const trigger = matButton._elementRef.nativeElement; @@ -189,7 +155,51 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } } - getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { + private updateErrorText(): void { + if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE + && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { + this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; + } else if (!this.argumentsFormArray.controls.length) { + this.errorText = 'calculated-fields.hint.arguments-empty'; + } else { + this.errorText = ''; + } + } + + private getArgumentsObject(): Record { + return this.argumentsFormArray.controls.reduce((acc, control) => { + const rawValue = control.getRawValue(); + const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; + acc[argumentName] = argument; + return acc; + }, {} as Record); + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + this.populateArgumentsFormArray(argumentsObj) + } + + private populateArgumentsFormArray(argumentsObj: Record): void { + Object.keys(argumentsObj).forEach(key => { + this.argumentsFormArray.push(this.fb.group({ + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + ...argumentsObj[key], + ...(argumentsObj[key].refEntityId ? { + refEntityId: this.fb.group({ + entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], + id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], + }), + } : {}), + refEntityKey: this.fb.group({ + type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], + key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], + }), + }) as AbstractControl); + }); + } + + private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { return this.fb.group({ ...value, argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], @@ -205,9 +215,4 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }), }) } - - onDelete(index: number): void { - this.argumentsFormArray.removeAt(index); - this.argumentsFormArray.markAsDirty(); - } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 7d1373bebb..e6adc1b4d8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -68,7 +68,7 @@ formControlName="arguments" [entityId]="data.entityId" [tenantId]="data.tenantId" - [calculatedFieldType]="fieldFormGroup.get('type').valueChanges | async" + [calculatedFieldType]="fieldFormGroup.get('type').value" />
@@ -96,7 +96,7 @@ [functionArgs]="functionArgs$ | async" [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" - helpId="[TODO]: ADD VALID LINK HERE!!!" + helpId="[TODO]: [Calculated Fields] add valid link" /> }
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 99bf62ad46..7f9af5cead 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -47,7 +47,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent - @if (argumentFormGroup.get('argumentName').hasError('required') && argumentFormGroup.get('argumentName').touched) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('pattern') && argumentFormGroup.get('argumentName').touched) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('maxlength') && argumentFormGroup.get('argumentName').touched) { - - warning - + @if (argumentFormGroup.get('argumentName').touched) { + @if (argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } } @@ -118,7 +120,7 @@ @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
{{ 'calculated-fields.timeseries-key' | translate }}
- +
} @else {
@@ -143,6 +145,7 @@
{{ 'calculated-fields.attribute-key' | translate }}
this.updateEntityFilter(this.entityType)); @@ -196,6 +197,4 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme typeControl.updateValueAndValidity(); } } - - protected readonly ArgumentEntityType = ArgumentEntityType; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts index c3d1ede02e..bc89e4dc6f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -16,3 +16,4 @@ export * from './dialog/calculated-field-dialog.component'; export * from './arguments-table/calculated-field-arguments-table.component'; +export * from './panel/calculated-field-argument-panel.component'; diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index ab59f412cd..6a4f87ca6b 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -32,8 +32,7 @@ [entityName]="entity.name"> - + { - readonly EntityType = EntityType; - constructor(protected store: Store) { super(store); } @@ -37,4 +34,5 @@ export class DeviceTabsComponent extends EntityTabsComponent { ngOnInit() { super.ngOnInit(); } + } diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts index 2a33a74786..fc13238db0 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts @@ -47,9 +47,6 @@ import { EntityFilter } from '@shared/models/query/query.models'; multi: true } ], - host: { - class: 'w-full' - } }) export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Validator { @@ -63,7 +60,7 @@ export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Val searchText = ''; keyInputSubject = new Subject(); - private onChange: (value: string) => void; + private propagateChange: (value: string) => void; private cachedResult: EntitiesKeysByQuery; keys$ = this.keyInputSubject.asObservable() @@ -101,7 +98,7 @@ export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Val ) { this.keyControl.valueChanges .pipe(takeUntilDestroyed()) - .subscribe(value => this.onChange(value)); + .subscribe(value => this.propagateChange(value)); effect(() => { if (this.keyScopeType() || this.entityFilter() && this.dataKeyType()) { this.cachedResult = null; @@ -119,7 +116,7 @@ export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Val } registerOnChange(onChange: (value: string) => void): void { - this.onChange = onChange; + this.propagateChange = onChange; } registerOnTouched(_): void {} diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 73b3ecd861..54798a23e7 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -25,6 +25,7 @@ export interface CalculatedField extends Omit, 'labe externalId?: string; configuration: CalculatedFieldConfiguration; type: CalculatedFieldType; + entityId: EntityId; } export enum CalculatedFieldType { From d3216f3ee78ce154ea3a4c8a224003728497c84f Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 13:11:12 +0200 Subject: [PATCH 148/416] Changed table load --- .../calculated-fields-table.component.ts | 62 ++++++------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 853870982a..4c22ad2896 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -18,8 +18,8 @@ import { ChangeDetectionStrategy, Component, DestroyRef, - Input, - OnInit, + effect, + input, ViewChild, } from '@angular/core'; import { EntityId } from '@shared/models/id/entity-id'; @@ -39,36 +39,14 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; styleUrls: ['./calculated-fields-table.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush, }) -export class CalculatedFieldsTableComponent implements OnInit { - - @Input() - set entityId(entityId: EntityId) { - if (this.entityIdValue !== entityId) { - this.entityIdValue = entityId; - if (!this.activeValue) { - this.hasInitialized = true; - } - } - } - - @Input() - set active(active: boolean) { - if (this.activeValue !== active) { - this.activeValue = active; - if (this.activeValue && this.hasInitialized) { - this.hasInitialized = false; - this.entitiesTable.updateData(); - } - } - } +export class CalculatedFieldsTableComponent { @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; - calculatedFieldsTableConfig: CalculatedFieldsTableConfig; + active = input(); + entityId = input(); - private activeValue = false; - private hasInitialized = false; - private entityIdValue: EntityId; + calculatedFieldsTableConfig: CalculatedFieldsTableConfig; constructor(private calculatedFieldsService: CalculatedFieldsService, private translate: TranslateService, @@ -77,20 +55,20 @@ export class CalculatedFieldsTableComponent implements OnInit { private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, private destroyRef: DestroyRef) { - } - ngOnInit() { - this.hasInitialized = !this.activeValue; - - this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( - this.calculatedFieldsService, - this.translate, - this.dialog, - this.entityIdValue, - this.store, - this.durationLeft, - this.popoverService, - this.destroyRef, - ); + effect(() => { + if (this.active()) { + this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( + this.calculatedFieldsService, + this.translate, + this.dialog, + this.entityId(), + this.store, + this.durationLeft, + this.popoverService, + this.destroyRef, + ); + } + }); } } From 2aeacf80c54f78e2793ce0b2e4066f8d045e1d88 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 31 Jan 2025 13:26:52 +0200 Subject: [PATCH 149/416] Clean up upgrade for next version --- .../main/data/upgrade/basic/schema_update.sql | 195 ------------------ .../install/ThingsboardInstallService.java | 19 +- .../service/install/InstallScripts.java | 20 -- .../update/DefaultDataUpdateService.java | 4 - 4 files changed, 3 insertions(+), 235 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 64bbbaca05..6b87dc6dde 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -14,198 +14,3 @@ -- limitations under the License. -- -ALTER TABLE user_credentials ADD COLUMN IF NOT EXISTS last_login_ts BIGINT; -UPDATE user_credentials c SET last_login_ts = (SELECT (additional_info::json ->> 'lastLoginTs')::bigint FROM tb_user u WHERE u.id = c.user_id) - WHERE last_login_ts IS NULL; - -ALTER TABLE user_credentials ADD COLUMN IF NOT EXISTS failed_login_attempts INT; -UPDATE user_credentials c SET failed_login_attempts = (SELECT (additional_info::json ->> 'failedLoginAttempts')::int FROM tb_user u WHERE u.id = c.user_id) - WHERE failed_login_attempts IS NULL; - -UPDATE tb_user SET additional_info = (additional_info::jsonb - 'lastLoginTs' - 'failedLoginAttempts' - 'userCredentialsEnabled')::text - WHERE additional_info IS NOT NULL AND additional_info != 'null' AND jsonb_typeof(additional_info::jsonb) = 'object'; - --- UPDATE RULE NODE DEBUG MODE TO DEBUG STRATEGY START - -ALTER TABLE rule_node ADD COLUMN IF NOT EXISTS debug_settings varchar(1024) DEFAULT null; -DO -$$ - BEGIN - IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'rule_node' AND column_name = 'debug_mode') - THEN - UPDATE rule_node SET debug_settings = '{"failuresEnabled": true, "allEnabledUntil": ' || cast((extract(epoch from now()) + 900) * 1000 as bigint) || '}' WHERE debug_mode = true; -- 15 minutes according to thingsboard.yml default settings. - ALTER TABLE rule_node DROP COLUMN debug_mode; - END IF; - END -$$; - --- UPDATE RULE NODE DEBUG MODE TO DEBUG STRATEGY END - - --- CREATE MOBILE APP BUNDLES FROM EXISTING APPS - -CREATE TABLE IF NOT EXISTS mobile_app_bundle ( - id uuid NOT NULL CONSTRAINT mobile_app_bundle_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid, - title varchar(255), - description varchar(1024), - android_app_id uuid UNIQUE, - ios_app_id uuid UNIQUE, - layout_config varchar(16384), - oauth2_enabled boolean, - CONSTRAINT fk_android_app_id FOREIGN KEY (android_app_id) REFERENCES mobile_app(id) ON DELETE SET NULL, - CONSTRAINT fk_ios_app_id FOREIGN KEY (ios_app_id) REFERENCES mobile_app(id) ON DELETE SET NULL -); -CREATE INDEX IF NOT EXISTS mobile_app_bundle_tenant_id ON mobile_app_bundle(tenant_id); - -ALTER TABLE mobile_app ADD COLUMN IF NOT EXISTS platform_type varchar(32), - ADD COLUMN IF NOT EXISTS status varchar(32), - ADD COLUMN IF NOT EXISTS version_info varchar(100000), - ADD COLUMN IF NOT EXISTS store_info varchar(16384), - DROP CONSTRAINT IF EXISTS mobile_app_pkg_name_key, - DROP CONSTRAINT IF EXISTS mobile_app_unq_key; - --- rename mobile_app_oauth2_client to mobile_app_bundle_oauth2_client -DO -$$ - BEGIN - -- in case of running the upgrade script a second time - IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'mobile_app_oauth2_client') THEN - ALTER TABLE mobile_app_oauth2_client RENAME TO mobile_app_bundle_oauth2_client; - ALTER TABLE mobile_app_bundle_oauth2_client DROP CONSTRAINT IF EXISTS fk_domain; - ALTER TABLE mobile_app_bundle_oauth2_client RENAME COLUMN mobile_app_id TO mobile_app_bundle_id; - END IF; - END; -$$; - - -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - --- duplicate each mobile app and create mobile app bundle for the pair of android and ios app -DO -$$ - DECLARE - generatedBundleId uuid; - iosAppId uuid; - mobileAppRecord RECORD; - BEGIN - -- in case of running the upgrade script a second time - IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'mobile_app' and column_name = 'oauth2_enabled') THEN - UPDATE mobile_app SET platform_type = 'ANDROID' WHERE platform_type IS NULL; - UPDATE mobile_app SET status = 'DRAFT' WHERE mobile_app.status IS NULL; - FOR mobileAppRecord IN SELECT * FROM mobile_app - LOOP - -- duplicate app for iOS platform type - iosAppId := uuid_generate_v4(); - INSERT INTO mobile_app(id, created_time, tenant_id, pkg_name, app_secret, platform_type, status) - VALUES (iosAppId, mobileAppRecord.created_time, mobileAppRecord.tenant_id, mobileAppRecord.pkg_name, mobileAppRecord.app_secret, 'IOS', mobileAppRecord.status) - ON CONFLICT DO NOTHING; - -- create bundle for android and iOS app - generatedBundleId := uuid_generate_v4(); - INSERT INTO mobile_app_bundle(id, created_time, tenant_id, title, android_app_id, ios_app_id, oauth2_enabled) - VALUES (generatedBundleId, mobileAppRecord.created_time, mobileAppRecord.tenant_id, - mobileAppRecord.pkg_name || ' (autogenerated)', mobileAppRecord.id, iosAppId, mobileAppRecord.oauth2_enabled) - ON CONFLICT DO NOTHING; - UPDATE mobile_app_bundle_oauth2_client SET mobile_app_bundle_id = generatedBundleId WHERE mobile_app_bundle_id = mobileAppRecord.id; - END LOOP; - END IF; - IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'fk_mobile_app_bundle_oauth2_client_bundle_id') THEN - ALTER TABLE mobile_app_bundle_oauth2_client ADD CONSTRAINT fk_mobile_app_bundle_oauth2_client_bundle_id - FOREIGN KEY (mobile_app_bundle_id) REFERENCES mobile_app_bundle(id) ON DELETE CASCADE; - END IF; - ALTER TABLE mobile_app DROP COLUMN IF EXISTS oauth2_enabled; - IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'mobile_app_pkg_name_platform_unq_key') THEN - ALTER TABLE mobile_app ADD CONSTRAINT mobile_app_pkg_name_platform_unq_key UNIQUE (pkg_name, platform_type); - END IF; - END; -$$; - -ALTER TABLE IF EXISTS mobile_app_settings RENAME TO qr_code_settings; -ALTER TABLE qr_code_settings ADD COLUMN IF NOT EXISTS mobile_app_bundle_id uuid, - ADD COLUMN IF NOT EXISTS android_enabled boolean, - ADD COLUMN IF NOT EXISTS ios_enabled boolean; - --- migrate mobile apps from qr code settings to mobile_app, create mobile app bundle for the pair of apps -DO -$$ - DECLARE - androidPkgName varchar; - iosPkgName varchar; - androidAppId uuid; - iosAppId uuid; - generatedBundleId uuid; - qrCodeRecord RECORD; - BEGIN - -- in case of running the upgrade script a second time - IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'qr_code_settings' AND column_name = 'android_config') THEN - FOR qrCodeRecord IN SELECT * FROM qr_code_settings - LOOP - generatedBundleId := NULL; - -- migrate android config - IF (qrCodeRecord.android_config::jsonb ->> 'appPackage' IS NOT NULL) THEN - androidPkgName := qrCodeRecord.android_config::jsonb ->> 'appPackage'; - SELECT id into androidAppId FROM mobile_app WHERE pkg_name = androidPkgName AND platform_type = 'ANDROID'; - IF androidAppId IS NULL THEN - androidAppId := uuid_generate_v4(); - INSERT INTO mobile_app(id, created_time, tenant_id, pkg_name, platform_type, status, store_info) - VALUES (androidAppId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, - androidPkgName, 'ANDROID', 'DRAFT', qrCodeRecord.android_config::jsonb - 'appPackage' - 'enabled'); - generatedBundleId := uuid_generate_v4(); - INSERT INTO mobile_app_bundle(id, created_time, tenant_id, title, android_app_id) - VALUES (generatedBundleId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, androidPkgName || ' (autogenerated)', androidAppId); - UPDATE qr_code_settings SET mobile_app_bundle_id = generatedBundleId; - ELSE - UPDATE mobile_app SET store_info = qrCodeRecord.android_config::jsonb - 'appPackage' - 'enabled' WHERE id = androidAppId; - UPDATE qr_code_settings SET mobile_app_bundle_id = (SELECT id FROM mobile_app_bundle WHERE mobile_app_bundle.android_app_id = androidAppId); - END IF; - END IF; - UPDATE qr_code_settings SET android_enabled = (qrCodeRecord.android_config::jsonb ->> 'enabled')::boolean WHERE id = qrCodeRecord.id; - - -- migrate ios config - IF (qrCodeRecord.ios_config::jsonb ->> 'appId' IS NOT NULL) THEN - iosPkgName := substring(qrCodeRecord.ios_config::jsonb ->> 'appId', strpos(qrCodeRecord.ios_config::jsonb ->> 'appId', '.') + 1); - SELECT id INTO iosAppId FROM mobile_app WHERE pkg_name = iosPkgName AND platform_type = 'IOS'; - IF iosAppId IS NULL THEN - iosAppId := uuid_generate_v4(); - INSERT INTO mobile_app(id, created_time, tenant_id, pkg_name, platform_type, status, store_info) - VALUES (iosAppId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, - iosPkgName, 'IOS', 'DRAFT', qrCodeRecord.ios_config::jsonb - 'enabled'); - IF generatedBundleId IS NULL THEN - generatedBundleId := uuid_generate_v4(); - INSERT INTO mobile_app_bundle(id, created_time, tenant_id, title, ios_app_id) - VALUES (generatedBundleId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, iosPkgName || ' (autogenerated)', iosAppId); - UPDATE qr_code_settings SET mobile_app_bundle_id = generatedBundleId; - ELSE - UPDATE mobile_app_bundle SET ios_app_id = iosAppId WHERE id = generatedBundleId; - END IF; - ELSE - UPDATE qr_code_settings SET mobile_app_bundle_id = (SELECT id FROM mobile_app_bundle WHERE mobile_app_bundle.ios_app_id = iosAppId); - UPDATE mobile_app SET store_info = qrCodeRecord.ios_config::jsonb - 'enabled' WHERE id = iosAppId; - END IF; - END IF; - UPDATE qr_code_settings SET ios_enabled = (qrCodeRecord.ios_config::jsonb -> 'enabled')::boolean WHERE id = qrCodeRecord.id; - END LOOP; - ALTER TABLE qr_code_settings RENAME CONSTRAINT mobile_app_settings_tenant_id_unq_key TO qr_code_settings_tenant_id_unq_key; - ALTER TABLE qr_code_settings RENAME CONSTRAINT mobile_app_settings_pkey TO qr_code_settings_pkey; - END IF; - ALTER TABLE qr_code_settings DROP COLUMN IF EXISTS android_config, DROP COLUMN IF EXISTS ios_config; - END; -$$; - --- update constraint name -DO -$$ - BEGIN - ALTER TABLE domain DROP CONSTRAINT IF EXISTS domain_unq_key; - IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'domain_name_key') THEN - ALTER TABLE domain ADD CONSTRAINT domain_name_key UNIQUE (name); - END IF; - END; -$$; - --- UPDATE RESOURCE JS_MODULE SUB TYPE START - -UPDATE resource SET resource_sub_type = 'EXTENSION' WHERE resource_type = 'JS_MODULE' AND resource_sub_type IS NULL; - --- UPDATE RESOURCE JS_MODULE SUB TYPE END diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 622c0ab2dc..ea4c059222 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -35,8 +35,6 @@ import org.thingsboard.server.service.install.migrate.TsLatestMigrateService; import org.thingsboard.server.service.install.update.CacheCleanupService; import org.thingsboard.server.service.install.update.DataUpdateService; -import static org.thingsboard.server.service.install.update.DefaultDataUpdateService.getEnv; - @Service @Profile("install") @Slf4j @@ -99,8 +97,6 @@ public class ThingsboardInstallService { if ("cassandra-latest-to-postgres".equals(upgradeFromVersion)) { log.info("Migrating ThingsBoard latest timeseries data from cassandra to SQL database ..."); latestMigrateService.migrate(); - } else if (upgradeFromVersion.equals("3.9.0-resources")) { - installScripts.updateResourcesUsage(); } else { // TODO DON'T FORGET to update SUPPORTED_VERSIONS_FROM in DefaultDatabaseSchemaSettingsService databaseSchemaVersionService.validateSchemaSettings(); @@ -118,25 +114,16 @@ public class ThingsboardInstallService { entityDatabaseSchemaService.createOrUpdateDeviceInfoView(persistToTelemetry); // Creates missing indexes. entityDatabaseSchemaService.createDatabaseIndexes(); - // Runs upgrade scripts that are not possible in plain SQL. + // TODO: cleanup update code after each release - if (!getEnv("SKIP_RESOURCES_USAGE_MIGRATION", false)) { - installScripts.setUpdateResourcesUsage(true); - } else { - log.info("Skipping resources usage migration. Run the upgrade with fromVersion as '3.9.0-resources' to migrate"); - } - if (installScripts.isUpdateResourcesUsage()) { - installScripts.updateResourcesUsage(); - } + + // Runs upgrade scripts that are not possible in plain SQL. dataUpdateService.updateData(); log.info("Updating system data..."); dataUpdateService.upgradeRuleNodes(); systemDataLoaderService.loadSystemWidgets(); installScripts.loadSystemLwm2mResources(); installScripts.loadSystemImagesAndResources(); - if (installScripts.isUpdateImages()) { - installScripts.updateImages(); - } databaseSchemaVersionService.updateSchemaVersion(); } log.info("Upgrade finished successfully!"); diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index a6ccb85e11..b230be0d57 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -17,8 +17,6 @@ package org.thingsboard.server.service.install; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -120,11 +118,6 @@ public class InstallScripts { @Autowired private ResourcesUpdater resourcesUpdater; - @Getter @Setter - private boolean updateImages = false; - - @Getter @Setter - private boolean updateResourcesUsage = false; @Autowired private ImageService imageService; @@ -395,14 +388,6 @@ public class InstallScripts { } } - public void updateImages() { - resourcesUpdater.updateWidgetsBundlesImages(); - resourcesUpdater.updateWidgetTypesImages(); - resourcesUpdater.updateDashboardsImages(); - resourcesUpdater.updateDeviceProfilesImages(); - resourcesUpdater.updateAssetProfilesImages(); - } - public void loadSystemImagesAndResources() { log.info("Loading system images and resources..."); Stream dashboardsFiles = Stream.concat(listDir(Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR)), @@ -512,11 +497,6 @@ public class InstallScripts { } } - public void updateResourcesUsage() { - resourcesUpdater.updateDashboardsResources(); - resourcesUpdater.updateWidgetsResources(); - } - private void loadSystemResources(Path dir, ResourceType resourceType, ResourceSubType resourceSubType) { listDir(dir).forEach(resourceFile -> { String resourceKey = resourceFile.getFileName().toString(); diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index e8622fae37..cea938bce9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -34,7 +34,6 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.RuleNodeClassInfo; -import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.utils.TbNodeUpgradeUtils; import java.util.ArrayList; @@ -58,9 +57,6 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired JpaExecutorService jpaExecutorService; - @Autowired - private InstallScripts installScripts; - @Override public void updateData() throws Exception { log.info("Updating data ..."); From 83b338c697ff6ab7188886e273df4736dd1c3d80 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 31 Jan 2025 13:54:53 +0200 Subject: [PATCH 150/416] Entity lifecycle implementation --- .../server/actors/app/AppActor.java | 5 +- .../CalculatedFieldEntityActor.java | 6 + .../CalculatedFieldEntityDeleteMsg.java | 44 +++ ...CalculatedFieldEntityMessageProcessor.java | 51 ++- .../CalculatedFieldManagerActor.java | 8 +- ...alculatedFieldManagerMessageProcessor.java | 215 +++++++++++- .../EntityInitCalculatedFieldMsg.java | 41 +++ .../server/actors/tenant/TenantActor.java | 3 +- .../cf/CalculatedFieldExecutionService.java | 21 +- ...efaultCalculatedFieldExecutionService.java | 314 +++--------------- .../cf/ctx/CalculatedFieldStateService.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 9 +- .../cf/ctx/state/RocksDBStateService.java | 4 +- .../entitiy/EntityStateSourcingListener.java | 10 +- ...faultTbCalculatedFieldConsumerService.java | 100 +----- .../queue/DefaultTbClusterService.java | 150 ++++----- .../server/cluster/TbClusterService.java | 4 +- .../server/queue/TbQueueCallback.java | 14 + .../server/common/msg/MsgType.java | 14 +- .../cf/CalculatedFieldEntityLifecycleMsg.java | 37 +++ .../msg/plugin/ComponentLifecycleMsg.java | 20 ++ .../server/common/util/ProtoUtils.java | 41 ++- common/proto/src/main/proto/queue.proto | 35 +- 23 files changed, 595 insertions(+), 553 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 9124dc0a9a..8a6907c107 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -115,12 +115,13 @@ public class AppActor extends ContextAwareActor { case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: - case CF_UPDATE_MSG: + case CF_ENTITY_LIFECYCLE_MSG: + //TODO: use priority from the message body. For example, messages about CF lifecycle are important and Device lifecycle are not. + // same for the Linked telemetry. onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); break; case CF_TELEMETRY_MSG: case CF_LINKED_TELEMETRY_MSG: - case CF_ENTITY_UPDATE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); break; default: diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index a5461c6152..cebe6b6a60 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -53,6 +53,12 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { case CF_STATE_RESTORE_MSG: processor.process((CalculatedFieldStateRestoreMsg) msg); break; + case CF_ENTITY_INIT_CF_MSG: + processor.process((EntityInitCalculatedFieldMsg) msg); + break; + case CF_ENTITY_DELETE_MSG: + processor.process((CalculatedFieldEntityDeleteMsg) msg); + break; case CF_ENTITY_TELEMETRY_MSG: processor.process((EntityCalculatedFieldTelemetryMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java new file mode 100644 index 0000000000..8df6a63a82 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +public class CalculatedFieldEntityDeleteMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final TbCallback callback; + + public CalculatedFieldEntityDeleteMsg(TenantId tenantId, + EntityId entityId, + TbCallback callback) { + this.tenantId = tenantId; + this.entityId = entityId; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_DELETE_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index cea3cce791..b8f357fd87 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -88,6 +89,29 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM states.put(msg.getId().cfId(), msg.getState()); } + public void process(EntityInitCalculatedFieldMsg msg) { + var cfCtx = msg.getCtx(); + if (msg.isForceReinit()) { + states.remove(cfCtx.getCfId()); + } + var cfState = getOrInitState(cfCtx); + processStateIfReady(cfCtx, Collections.singletonList(cfCtx.getCfId()), cfState, msg.getCallback()); + } + + public void process(CalculatedFieldEntityDeleteMsg msg) { + if (this.entityId.equals(msg.getEntityId())) { + MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); + states.forEach((cfId, state) -> cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); + ctx.stop(ctx.getSelf()); + } else { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var state = states.remove(cfId); + if (state != null) { + cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + } + } + } + public void process(EntityCalculatedFieldTelemetryMsg msg) { var proto = msg.getProto(); var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); @@ -152,15 +176,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } CalculatedFieldState state = getOrInitState(ctx); if (state.updateState(newArgValues)) { - if (state.isReady()) { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); - cfIdList = new ArrayList<>(cfIdList); - cfIdList.add(ctx.getCfId()); - cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); - } else { - callback.onSuccess(); // State was updated but no calculation performed; - } - cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + processStateIfReady(ctx, cfIdList, state, callback); } else { callback.onSuccess(CALLBACKS_PER_CF); } @@ -183,13 +201,24 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return state; } + @SneakyThrows + private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, TbCallback callback) { + if (state.isReady()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + } else { + callback.onSuccess(); // State was updated but no calculation performed; + } + cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + } + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { return mapToArguments(ctx.getMainEntityArguments(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, data); @@ -221,7 +250,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, scope, attrDataList); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 1c198c660d..22909dd5af 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -22,6 +22,7 @@ import org.thingsboard.server.actors.TbActorException; import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; @@ -66,8 +67,8 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { case CF_STATE_RESTORE_MSG: processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); break; - case CF_UPDATE_MSG: -// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); + case CF_ENTITY_LIFECYCLE_MSG: + processor.onEntityLifecycleMsg((CalculatedFieldEntityLifecycleMsg) msg); break; case CF_TELEMETRY_MSG: processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); @@ -75,9 +76,6 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { case CF_LINKED_TELEMETRY_MSG: processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); break; - case CF_ENTITY_UPDATE_MSG: -// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); - break; default: return false; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4bce7cb322..e82f407e3a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,16 +16,13 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -36,18 +33,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.queue.discovery.HashPartitionService; -import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -78,7 +70,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private final CalculatedFieldExecutionService cfService; + private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; @@ -88,7 +81,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); - this.cfService = systemContext.getCalculatedFieldExecutionService(); + this.cfExecService = systemContext.getCalculatedFieldExecutionService(); + this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.tenantId = tenantId; @@ -104,7 +98,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cfCtx); msg.getCallback().onSuccess(); } @@ -112,7 +106,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var link = msg.getLink(); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link); msg.getCallback().onSuccess(); } @@ -124,6 +118,168 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } + public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) { + var entityType = msg.getData().getEntityId().getEntityType(); + var event = msg.getData().getEvent(); + switch (entityType) { + case CALCULATED_FIELD: { + switch (event) { + case CREATED: + onCfCreated(msg.getData(), msg.getCallback()); + break; + case UPDATED: + onCfUpdated(msg.getData(), msg.getCallback()); + break; + case DELETED: + onCfDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } + case DEVICE: + case ASSET: { + switch (event) { + case CREATED: + onEntityCreated(msg.getData(), msg.getCallback()); + break; + case UPDATED: + onEntityUpdated(msg.getData(), msg.getCallback()); + break; + case DELETED: + onEntityDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } + default: { + msg.getCallback().onSuccess(); + } + } + } + + private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { + EntityId entityId = msg.getEntityId(); + var entityIdFields = getCalculatedFieldsByEntityId(entityId); + var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + var fieldsCount = entityIdFields.size() + profileIdFields.size(); + if (fieldsCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + } else { + callback.onSuccess(); + } + } + + private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + if (msg.getOldProfileId() != null && msg.getOldProfileId() != msg.getProfileId()) { + var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); + var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); + var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); + if (fieldsCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + var entityId = msg.getEntityId(); + oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), callback)); + newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + } else { + callback.onSuccess(); + } + } + } + + private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } + + private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + if (calculatedFields.containsKey(cfId)) { + log.warn("[{}] CF was already initialized [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var cf = cfDaoService.findById(msg.getTenantId(), cfId); + if (cf == null) { + log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + calculatedFields.put(cf.getId(), cfCtx); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + initCf(cfCtx, callback, false); + } + } + } + + private void onCfUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var oldCfCtx = calculatedFields.get(cfId); + if (oldCfCtx == null) { + onCfCreated(msg, callback); + } else { + var newCf = cfDaoService.findById(msg.getTenantId(), cfId); + if (newCf == null) { + log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService()); + calculatedFields.put(newCf.getId(), newCfCtx); + List oldCfList = entityIdCalculatedFields.get(newCf.getId()); + List newCfList = new ArrayList<>(oldCfList.size()); + boolean found = false; + for (CalculatedFieldCtx oldCtx : oldCfList) { + if (oldCtx.getCfId().equals(newCf.getId())) { + newCfList.add(newCfCtx); + found = true; + } else { + newCfList.add(oldCtx); + } + } + if (!found) { + newCfList.add(newCfCtx); + } + entityIdCalculatedFields.put(newCf.getId(), newCfList); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + if (newCfCtx.hasSignificantChanges(oldCfCtx)) { + initCf(newCfCtx, callback, true); + } + } + } + + } + + private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var cfCtx = calculatedFields.remove(cfId); + if (cfCtx == null) { + log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + EntityId entityId = cfCtx.getEntityId(); + EntityType entityType = cfCtx.getEntityId().getEntityType(); + if (isProfileEntity(entityType)) { + var entityIds = getEntitiesByProfile(entityId); + if (!entityIds.isEmpty()) { + //TODO: no need to do this if we cache all created actors and know which one belong to us; + var multiCallback = new MultipleTbCallback(entityIds.size(), callback); + entityIds.forEach(id -> deleteCfForEntity(entityId, cfId, multiCallback)); + } else { + callback.onSuccess(); + } + } else { + deleteCfForEntity(entityId, cfId, callback); + } + } + } + public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); // 2 = 1 for CF processing + 1 for links processing @@ -140,7 +296,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware List linkedCalculatedFields = filterCalculatedFieldLinks(msg); var linksSize = linkedCalculatedFields.size(); if (linksSize > 0) { - cfService.pushMsgToLinks(msg, linkedCalculatedFields, callback); + cfExecService.pushMsgToLinks(msg, linkedCalculatedFields, callback); } else { callback.onSuccess(); } @@ -237,6 +393,33 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return entities; } + private void initCf(CalculatedFieldCtx cfCtx, TbCallback callback, boolean forceStateReinit) { + EntityId entityId = cfCtx.getEntityId(); + EntityType entityType = cfCtx.getEntityId().getEntityType(); + if (isProfileEntity(entityType)) { + var entityIds = getEntitiesByProfile(entityId); + if (!entityIds.isEmpty()) { + var multiCallback = new MultipleTbCallback(entityIds.size(), callback); + entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); + } else { + callback.onSuccess(); + } + } else { + initCfForEntity(entityId, cfCtx, forceStateReinit, callback); + } + } + + private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { + getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); + } + + private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { + getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); + } + + private static boolean isProfileEntity(EntityType entityType) { + return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); + } private EntityId getProfileId(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java new file mode 100644 index 0000000000..00a378c90a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + private final boolean forceReinit; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_INIT_CF_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 6ae10aaece..1b008f462a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -170,12 +170,11 @@ public class TenantActor extends RuleChainManagerActor { case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: - case CF_UPDATE_MSG: + case CF_ENTITY_LIFECYCLE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); break; case CF_TELEMETRY_MSG: case CF_LINKED_TELEMETRY_MSG: - case CF_ENTITY_UPDATE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index d8daae2e64..19f60165cf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -20,16 +20,11 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -50,25 +45,11 @@ public interface CalculatedFieldExecutionService { void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto); - ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); -// void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); - - /* ===================================================== */ - - void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); - - void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback); - - void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback); - - void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); - - + void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 19561b73b9..9aca0d5b3f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,7 +39,6 @@ import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -65,7 +64,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; @@ -75,11 +73,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; -import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @@ -122,7 +117,6 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -273,6 +267,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas stateService.persistState(stateId, state, callback); } + @Override + public void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback) { + stateService.removeState(calculatedFieldEntityCtxId, callback); + } + @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); @@ -330,271 +329,44 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); } - @Override - public void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto) { - EntityId entityId = calculatedField.getEntityId(); - ToCalculatedFieldMsg msg = ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(proto).build(); - switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); - clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null); - } - case ASSET_PROFILE, DEVICE_PROFILE -> { - Set tpiSet = calculatedFieldCache.getEntitiesByProfile(calculatedField.getTenantId(), entityId).stream() - .map(targetEntityId -> partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, targetEntityId)) - .collect(Collectors.toSet()); - tpiSet.forEach(tpi -> clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null)); - } - default -> throw new IllegalArgumentException("Entity type '" + calculatedField.getId().getEntityType() - + "' does not support calculated fields."); - } - } - - @Override - public void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - ComponentLifecycleEvent event = proto.getEvent(); - if (ComponentLifecycleEvent.DELETED.equals(event)) { - log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - calculatedFieldCache.evict(calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); - callback.onSuccess(); - } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (ComponentLifecycleEvent.UPDATED.equals(event)) { - log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); - if (!shouldReinit) { - return; - } - } - if (cf != null) { - calculatedFieldCache.addCalculatedField(tenantId, calculatedFieldId); - EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(calculatedFieldCtx, entityId, callback); - } - case ASSET_PROFILE, DEVICE_PROFILE -> { - log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> entry.getValue().getRefEntityId() != null) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { - calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { - initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); - }); - }); - } - default -> - throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); - } - } else { - //Calculated field was probably deleted while message was in queue; - log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); - callback.onSuccess(); - } - callback.onSuccess(); - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); - } catch (Exception e) { - log.trace("Failed to process calculated field msg: [{}]", proto, e); - callback.onFailure(e); - } - } - - private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); - } else { - calculatedFieldCache.updateCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); - callback.onSuccess(); - shouldReinit = false; - } - return shouldReinit; - } - - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> { - if (ctxId.cfId().equals(calculatedFieldId)) { - stateService.removeState(ctxId); - return true; - } - return false; - }); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - boolean argumentsChanged = !oldCalculatedField.getConfiguration().getArguments().equals(newCalculatedField.getConfiguration().getArguments()); - - return entityIdChanged || typeChanged || argumentsChanged; - } - - @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); - EntityId entityId = request.getEntityId(); - - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - TenantId tenantId = request.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - - if (tpi.isMyPartition()) { - - processCalculatedFields(request, entityId); - processCalculatedFields(request, getProfileId(tenantId, entityId)); - - Map> tpiStatesToUpdate = new HashMap<>(); - processCalculatedFieldLinks(request, tpiStatesToUpdate); - if (!tpiStatesToUpdate.isEmpty()) { - tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(proto, ctxIds); - clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); - }); - } - } else { - clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(proto).build(), null); - } - } - } catch (Exception e) { - log.trace("Failed to update telemetry.", e); - } - } - - private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { - if (cfTargetEntityId != null) { - calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId).forEach(ctx -> { - Map updatedTelemetry = request.getMappedTelemetry(ctx, cfTargetEntityId); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = isProfileEntity(cfTargetEntityId) ? request.getEntityId() : cfTargetEntityId; - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } - } - - private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - - calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) - .forEach(link -> { - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - EntityId targetEntityId = ctx.getEntityId(); - - if (isProfileEntity(targetEntityId)) { - calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); - }); - } else { - processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); - } - }); - } - - private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); - if (targetEntityTpi.isMyPartition()) { - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, targetEntity, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - } else { - List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getTenantId(), ctx.getCfId(), targetEntity)); - } - } - - @Override - public void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto.getMsg()); - - if (proto.getLinksList().isEmpty()) { - onTelemetryUpdate(proto.getMsg(), callback); - return; - } - - proto.getLinksList().forEach(ctxIdProto -> { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } catch (Exception e) { - log.trace("Failed to process telemetry update msg: [{}]", proto, e); - } - } - - private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { - log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); - Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - -// updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); - } - - @Override - public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - if (proto.getDeleted()) { - log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); - - EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); - calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); - calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); - } - if (proto.getAdded()) { - log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); - - EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - initializeStateForEntityByProfile(entityId, newProfileId, callback); - } - if (proto.getUpdated()) { - log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); - - EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); - EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - - calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); - initializeStateForEntityByProfile(entityId, newProfileId, callback); - } - } else { - clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); - } - } catch (Exception e) { - log.trace("Failed to process entity update msg: [{}]", proto, e); - } - } +// @Override +// public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { +// try { +// TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); +// +// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); +// if (tpi.isMyPartition()) { +// log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); +// if (proto.getDeleted()) { +// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); +// +// EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); +// calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); +// calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); +// } +// if (proto.getAdded()) { +// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); +// +// EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); +// initializeStateForEntityByProfile(entityId, newProfileId, callback); +// } +// if (proto.getUpdated()) { +// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); +// +// EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); +// EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); +// +// calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); +// initializeStateForEntityByProfile(entityId, newProfileId, callback); +// } +// } else { +// clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); +// } +// } catch (Exception e) { +// log.trace("Failed to process entity update msg: [{}]", proto, e); +// } +// } private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index 2bf6b7e0f2..d37e7ebc3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -28,6 +28,6 @@ public interface CalculatedFieldStateService { void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - void removeState(CalculatedFieldEntityCtxId ctxId); + void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 37e402fbc8..6ca4369230 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; @@ -154,4 +153,12 @@ public class CalculatedFieldCtx { public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } + + public boolean hasSignificantChanges(CalculatedFieldCtx other) { + boolean entityIdChanged = !entityId.equals(other.entityId); + boolean typeChanged = !cfType.equals(other.cfType); + boolean argumentsChanged = !arguments.equals(other.arguments); + return entityIdChanged || typeChanged || argumentsChanged; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index aa82f2fc57..f5c882ea52 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -21,7 +21,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.RocksDBService; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -59,8 +58,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { } @Override - public void removeState(CalculatedFieldEntityCtxId ctxId) { + public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + callback.onSuccess(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 95b340c362..d40bf7b130 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -56,6 +56,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.TbQueueCallback; import java.util.Set; @@ -124,7 +125,7 @@ public class EntityStateSourcingListener { tbClusterService.onApiStateChange(apiUsageState, null); } case CALCULATED_FIELD -> { - onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity(), lifecycleEvent); + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); } default -> { } @@ -193,8 +194,7 @@ public class EntityStateSourcingListener { } case CALCULATED_FIELD -> { CalculatedField calculatedField = (CalculatedField) event.getEntity(); - ComponentLifecycleMsg lifecycleMsg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); - tbClusterService.onCalculatedFieldDeleted(calculatedField.getTenantId(), calculatedField, lifecycleMsg); + tbClusterService.onCalculatedFieldDeleted(calculatedField, null); } default -> { } @@ -276,13 +276,13 @@ public class EntityStateSourcingListener { } } - private void onCalculatedFieldUpdate(Object entity, Object oldEntity, ComponentLifecycleEvent lifecycleEvent) { + private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { CalculatedField calculatedField = (CalculatedField) entity; CalculatedField oldCalculatedField = null; if (oldEntity instanceof CalculatedField) { oldCalculatedField = (CalculatedField) oldEntity; } - tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), lifecycleEvent)); + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, TbQueueCallback.EMPTY); } private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 0fcc04bd5d..3d8bbddc80 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.queue; -import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; @@ -25,20 +24,17 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -70,6 +66,8 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.thingsboard.server.common.util.ProtoUtils.fromProto; + @Service @TbRuleEngineComponent @Slf4j @@ -86,8 +84,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer private final TbRuleEngineQueueFactory queueFactory; - private final CalculatedFieldExecutionService calculatedFieldExecutionService; - private MainQueueConsumerManager, CalculatedFieldQueueConfig> mainConsumer; private volatile ListeningExecutorService calculatedFieldsExecutor; @@ -101,12 +97,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer PartitionService partitionService, ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, - CalculatedFieldExecutionService calculatedFieldExecutionService, CalculatedFieldCache calculatedFieldCache) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; - this.calculatedFieldExecutionService = calculatedFieldExecutionService; } @PostConstruct @@ -168,12 +162,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); } else if (toCfMsg.hasComponentLifecycleMsg()) { log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); - /// TODO: forward to Actor system - forwardToCalculatedFieldService(toCfMsg.getComponentLifecycleMsg(), callback); - } else if (toCfMsg.hasEntityUpdateMsg()) { - log.trace("[{}] Forwarding entity update message for processing {}", id, toCfMsg.getEntityUpdateMsg()); - /// TODO: forward to Actor system - forwardToCalculatedFieldService(toCfMsg.getEntityUpdateMsg(), callback); + forwardToActorSystem(toCfMsg.getComponentLifecycleMsg(), callback); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -222,14 +211,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); - if (toCfNotification.hasComponentLifecycle()) { + if (toCfNotification.hasComponentLifecycleMsg()) { // from upstream (maybe removed since we don't need to init state for each partition) - forwardToActorSystem(toCfNotification.getComponentLifecycle(), callback); - handleComponentLifecycleMsg(id, ProtoUtils.fromProto(toCfNotification.getComponentLifecycle())); - } else if (toCfNotification.hasEntityUpdateMsg()) { - processEntityUpdateMsg(toCfNotification.getEntityUpdateMsg()); - // from upstream (maybe removed since we don't need to update state for each partition) - forwardToActorSystem(toCfNotification.getEntityUpdateMsg(), callback); + log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfNotification.getComponentLifecycleMsg()); + forwardToActorSystem(toCfNotification.getComponentLifecycleMsg(), callback); } else if (toCfNotification.hasLinkedTelemetryMsg()) { forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); } @@ -248,74 +233,9 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); } - private void forwardToCalculatedFieldService(ComponentLifecycleMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToActorSystem(ComponentLifecycleMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process entity updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToActorSystem(CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process entity updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void processEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto entityUpdateMsg) { - var tenantId = toTenantId(entityUpdateMsg.getTenantIdMSB(), entityUpdateMsg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityType(), new UUID(entityUpdateMsg.getEntityIdMSB(), entityUpdateMsg.getEntityIdLSB())); - if (entityUpdateMsg.getAdded()) { - var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); - calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); - } else if (entityUpdateMsg.getDeleted()) { - var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); - calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); - } else if (entityUpdateMsg.getUpdated()) { - var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); - var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); - calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); - } - } - - private void throwNotHandled(Object msg, TbCallback callback) { - log.warn("Message not handled: {}", msg); - callback.onFailure(new RuntimeException("Message not handled!")); + private void forwardToActorSystem(ComponentLifecycleMsgProto proto, TbCallback callback) { + var msg = fromProto(proto); + actorContext.tell(new CalculatedFieldEntityLifecycleMsg(msg.getTenantId(), msg, callback)); } private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2783e826fe..b272694b16 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -70,7 +70,6 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -362,8 +361,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); - producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toRuleEngineMsgs.incrementAndGet(); + pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, callback); } @Override @@ -435,7 +433,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - handleCalculatedFieldEntityDeleted(tenantId, deviceId, device.getDeviceProfileId()); + handleCalculatedFieldEntityDeleted(tenantId, deviceId); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -444,7 +442,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - handleCalculatedFieldEntityDeleted(tenantId, assetId, asset.getAssetProfileId()); + handleCalculatedFieldEntityDeleted(tenantId, assetId); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } @@ -597,9 +595,7 @@ public class DefaultTbClusterService implements TbClusterService { private void broadcast(ComponentLifecycleMsg msg) { ComponentLifecycleMsgProto componentLifecycleMsgProto = toProto(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); - TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); - Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); EntityType entityType = msg.getEntityId().getEntityType(); if (entityType.equals(EntityType.TENANT) || entityType.equals(EntityType.TENANT_PROFILE) @@ -622,14 +618,6 @@ public class DefaultTbClusterService implements TbClusterService { // No need to push notifications twice tbRuleEngineServices.removeAll(tbCoreServices); } - if (entityType.equals(EntityType.CALCULATED_FIELD)) { - for (String serviceId : tbCalculatedFieldServices) { - TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); - ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); - toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCfNotificationMsg), null); - toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS - } - } for (String serviceId : tbRuleEngineServices) { TopicPartitionInfo tpi = topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); @@ -669,56 +657,86 @@ public class DefaultTbClusterService implements TbClusterService { } @Override - public void onDeviceUpdated(Device device, Device old) { + public void onDeviceUpdated(Device entity, Device old) { var created = old == null; - broadcastEntityChangeToTransport(device.getTenantId(), device.getId(), device, null); + broadcastEntityChangeToTransport(entity.getTenantId(), entity.getId(), entity, null); if (old != null) { - boolean deviceNameChanged = !device.getName().equals(old.getName()); + boolean deviceNameChanged = !entity.getName().equals(old.getName()); if (deviceNameChanged) { - gatewayNotificationsService.onDeviceUpdated(device, old); + gatewayNotificationsService.onDeviceUpdated(entity, old); } - boolean deviceProfileChanged = !device.getDeviceProfileId().equals(old.getDeviceProfileId()); + boolean deviceProfileChanged = !entity.getDeviceProfileId().equals(old.getDeviceProfileId()); if (deviceProfileChanged) { - handleCalculatedFieldEntityUpdated(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.UPDATED) + .oldProfileId(old.getDeviceProfileId()) + .profileId(entity.getDeviceProfileId()) + .oldName(old.getName()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } if (deviceNameChanged || deviceProfileChanged) { - pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); + pushMsgToCore(new DeviceNameOrTypeUpdateMsg(entity.getTenantId(), entity.getId(), entity.getName(), entity.getType()), null); } } else { - handleCalculatedFieldEntityAdded(device.getTenantId(), device.getId(), device.getDeviceProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.CREATED) + .profileId(entity.getDeviceProfileId()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); - otaPackageStateService.update(device, old); + broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + sendDeviceStateServiceEvent(entity.getTenantId(), entity.getId(), created, !created, false); + otaPackageStateService.update(entity, old); } @Override - public void onAssetUpdated(Asset asset, Asset old) { + public void onAssetUpdated(Asset entity, Asset old) { var created = old == null; - broadcastEntityChangeToTransport(asset.getTenantId(), asset.getId(), asset, null); + broadcastEntityChangeToTransport(entity.getTenantId(), entity.getId(), entity, null); if (old != null) { - boolean assetTypeChanged = !asset.getType().equals(old.getType()); + boolean assetTypeChanged = !entity.getAssetProfileId().equals(old.getAssetProfileId()); if (assetTypeChanged) { - handleCalculatedFieldEntityUpdated(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.UPDATED) + .oldProfileId(old.getAssetProfileId()) + .profileId(entity.getAssetProfileId()) + .oldName(old.getName()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } } else { - handleCalculatedFieldEntityAdded(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.CREATED) + .profileId(entity.getAssetProfileId()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @Override - public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, ComponentLifecycleMsg lifecycleMsg) { - var created = oldCalculatedField == null; - calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); - broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback) { + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } @Override - public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, ComponentLifecycleMsg lifecycleMsg) { - CalculatedFieldId calculatedFieldId = calculatedField.getId(); - calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); - broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + public void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback) { + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), ComponentLifecycleEvent.DELETED); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } @Override @@ -849,54 +867,8 @@ public class DefaultTbClusterService implements TbClusterService { } } - private void handleCalculatedFieldEntityAdded(TenantId tenantId, EntityId entityId, EntityId newProfileId) { - handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, null, newProfileId, true, false, false); + private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId) { + ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.DELETED); + pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - - private void handleCalculatedFieldEntityUpdated(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, newProfileId, false, true, true); - } - - private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId, EntityId oldProfileId) { - handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, null, false, false, true); - } - - private void handleCalculatedFieldEntityUpdateEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId, boolean added, boolean updated, boolean deleted) { - CalculatedFieldEntityUpdateMsgProto.Builder builder = CalculatedFieldEntityUpdateMsgProto.newBuilder(); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - if (oldProfileId != null) { - builder.setEntityProfileType(oldProfileId.getEntityType().name()); - builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); - builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); - } - if (newProfileId != null) { - builder.setEntityProfileType(newProfileId.getEntityType().name()); - builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); - builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); - } - builder.setAdded(added); - builder.setUpdated(updated); - builder.setDeleted(deleted); - CalculatedFieldEntityUpdateMsgProto msg = builder.build(); - - broadcastEntityUpdateEvent(msg); - pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); - } - - private void broadcastEntityUpdateEvent(CalculatedFieldEntityUpdateMsgProto proto) { - TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); - Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); - Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); - for (String serviceId : tbCalculatedFieldServices) { - TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); - ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(); - toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), toCfNotificationMsg), null); - toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS - } - } - } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 588b135891..3d81de1cc4 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -131,8 +131,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); - void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, ComponentLifecycleMsg lifecycleMsg); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback); - void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, ComponentLifecycleMsg lifecycleMsg); + void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback); } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java index 7943c9d43d..9e8f8e4a0a 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java @@ -15,8 +15,22 @@ */ package org.thingsboard.server.queue; + public interface TbQueueCallback { + TbQueueCallback EMPTY = new TbQueueCallback() { + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + + } + + @Override + public void onFailure(Throwable t) { + + } + }; + void onSuccess(TbQueueMsgMetadata metadata); void onFailure(Throwable t); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 91419bee9c..e13e2d2ecb 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -135,14 +135,18 @@ public enum MsgType { EDGE_SYNC_REQUEST_TO_EDGE_SESSION_MSG, EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG, + CF_INIT_MSG, // Sent to init particular calculated field; CF_LINK_INIT_MSG, // Sent to init particular calculated field; - CF_STATE_RESTORE_MSG,// Sent to init particular calculated field entity state; - CF_TELEMETRY_MSG, + CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; + CF_ENTITY_LIFECYCLE_MSG, // Sent on CF/Device/Asset create/update/delete; + CF_TELEMETRY_MSG, // Sent from queue to actor system; + CF_LINKED_TELEMETRY_MSG, // Sent from queue to actor system; + + /* CF Manager Actor -> CF Entity actor */ CF_ENTITY_TELEMETRY_MSG, - CF_LINKED_TELEMETRY_MSG, - CF_UPDATE_MSG, - CF_ENTITY_UPDATE_MSG; + CF_ENTITY_INIT_CF_MSG, + CF_ENTITY_DELETE_MSG; @Getter private final boolean ignoreOnStart; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java new file mode 100644 index 0000000000..51eea1f943 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +public class CalculatedFieldEntityLifecycleMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final ComponentLifecycleMsg data; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_LIFECYCLE_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 000733cfd8..208b6ab2f5 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg.plugin; +import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; @@ -37,6 +38,25 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final TenantId tenantId; private final EntityId entityId; private final ComponentLifecycleEvent event; + private final String oldName; + private final String name; + private final EntityId oldProfileId; + private final EntityId profileId; + + public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { + this(tenantId, entityId, event, null, null, null, null); + } + + @Builder + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.event = event; + this.oldName = oldName; + this.name = name; + this.oldProfileId = oldProfileId; + this.profileId = profileId; + } public Optional getRuleChainId() { return entityId.getEntityType() == EntityType.RULE_CHAIN ? Optional.of((RuleChainId) entityId) : Optional.empty(); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 7ba11d8ea9..f0a34de08e 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -114,14 +114,28 @@ public class ProtoUtils { } public static TransportProtos.ComponentLifecycleMsgProto toProto(ComponentLifecycleMsg msg) { - return TransportProtos.ComponentLifecycleMsgProto.newBuilder() + var builder = TransportProtos.ComponentLifecycleMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) .setEntityType(toProto(msg.getEntityId().getEntityType())) .setEntityIdMSB(msg.getEntityId().getId().getMostSignificantBits()) .setEntityIdLSB(msg.getEntityId().getId().getLeastSignificantBits()) - .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())) - .build(); + .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())); + if (msg.getProfileId() != null) { + builder.setProfileIdMSB(msg.getProfileId().getId().getMostSignificantBits()); + builder.setProfileIdLSB(msg.getProfileId().getId().getLeastSignificantBits()); + } + if (msg.getOldProfileId() != null) { + builder.setProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); + builder.setProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); + } + if (msg.getName() != null) { + builder.setName(msg.getName()); + } + if (msg.getOldName() != null) { + builder.setName(msg.getOldName()); + } + return builder.build(); } public static TransportProtos.EntityTypeProto toProto(EntityType entityType) { @@ -129,11 +143,22 @@ public class ProtoUtils { } public static ComponentLifecycleMsg fromProto(TransportProtos.ComponentLifecycleMsgProto proto) { - return new ComponentLifecycleMsg( - TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), - ComponentLifecycleEvent.values()[proto.getEventValue()] - ); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + var builder = ComponentLifecycleMsg.builder() + .tenantId(TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()))) + .entityId(entityId) + .event(ComponentLifecycleEvent.values()[proto.getEventValue()]) + .name(proto.getName()) + .oldName(proto.getOldName()); + if (proto.getProfileIdMSB() != 0 || proto.getProfileIdLSB() != 0) { + var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; + builder.profileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB()))); + } + if (proto.getOldProfileIdMSB() != 0 || proto.getOldProfileIdLSB() != 0) { + var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; + builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); + } + return builder.build(); } public static EntityType fromProto(TransportProtos.EntityTypeProto entityType) { diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 69b912120e..18bcb7c6f4 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -773,22 +773,6 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } -message CalculatedFieldEntityUpdateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - string entityType = 3; - int64 entityIdMSB = 4; - int64 entityIdLSB = 5; - string entityProfileType = 6; - int64 oldProfileIdMSB = 7; - int64 oldProfileIdLSB = 8; - int64 newProfileIdMSB = 9; - int64 newProfileIdLSB = 10; - bool added = 11; - bool updated = 12; - bool deleted = 13; -} - message CalculatedFieldTelemetryMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -1203,6 +1187,13 @@ message ComponentLifecycleMsgProto { int64 entityIdMSB = 4; int64 entityIdLSB = 5; ComponentLifecycleEvent event = 6; + //Since 4.0. To replace the + string oldName = 7; + string name = 8; + int64 oldProfileIdMSB = 9; + int64 oldProfileIdLSB = 10; + int64 profileIdMSB = 11; + int64 profileIdLSB = 12; } message EdgeEventMsgProto { @@ -1630,16 +1621,14 @@ message ToEdgeEventNotificationMsg { } message ToCalculatedFieldMsg { - CalculatedFieldTelemetryMsgProto telemetryMsg = 1; - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; - ComponentLifecycleMsgProto componentLifecycleMsg = 3; - CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 4; + ComponentLifecycleMsgProto componentLifecycleMsg = 1; + CalculatedFieldTelemetryMsgProto telemetryMsg = 2; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; } message ToCalculatedFieldNotificationMsg { - ComponentLifecycleMsgProto componentLifecycle = 1; - CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 2; - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; + ComponentLifecycleMsgProto componentLifecycleMsg = 1; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; } /* Messages that are handled by ThingsBoard RuleEngine Service */ From bc835eb9d467f3bcff9777e86944171cdf64a852 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 15:33:30 +0200 Subject: [PATCH 151/416] Fixes --- .../calculated-fields-table-config.ts | 12 +- ...culated-field-arguments-table.component.ts | 2 +- .../calculated-field-dialog.component.ts | 23 +++- ...ulated-field-argument-panel.component.html | 122 +++++++++--------- ...ulated-field-argument-panel.component.scss | 4 + ...lculated-field-argument-panel.component.ts | 11 +- .../entity/entity-autocomplete.component.html | 2 +- .../entity-key-autocomplete.component.html | 12 +- 8 files changed, 111 insertions(+), 77 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index ae025a13f6..fcef11799d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -77,12 +77,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('name', 'common.name', '33%')); - this.columns.push( - new EntityTableColumn('type', 'common.type', '50px')); - this.columns.push( - new EntityTableColumn('expression', 'calculated-fields.expression', '50%', entity => entity.configuration.expression)); + const expressionColumn = new EntityTableColumn('expression', 'calculated-fields.expression', '33%', entity => entity.configuration?.expression); + expressionColumn.sortable = false; + + this.columns.push(new EntityTableColumn('name', 'common.name', '33%')); + this.columns.push(new EntityTableColumn('type', 'common.type', '50px')); + this.columns.push(expressionColumn); this.cellActionDescriptors.push( { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 1dedc5e80e..657b20abb4 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -133,7 +133,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }; this.keysPopupClosed = false; const argumentsPanelPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'leftBottom', false, null, + this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, ctx, {}, {}, {}, true); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 7f9af5cead..71fea36959 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -84,10 +84,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.toggleScopeByOutputType(type)); - this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); + this.observeTypeChanges(); } get configFormGroup(): FormGroup { @@ -113,10 +110,26 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.toggleScopeByOutputType(type)); + this.fieldFormGroup.get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleKeyByCalculatedFieldType(type)); } private toggleScopeByOutputType(type: OutputType): void { this.outputFormGroup.get('scope')[type === OutputType.Attribute? 'enable' : 'disable']({emitEvent: false}); } + + private toggleKeyByCalculatedFieldType(type: CalculatedFieldType): void { + this.outputFormGroup.get('name')[type === CalculatedFieldType.SIMPLE? 'enable' : 'disable']({emitEvent: false}); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 865b71406d..58923e0029 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -24,33 +24,35 @@
- @if (argumentFormGroup.get('argumentName').touched) { - @if (argumentFormGroup.get('argumentName').hasError('required')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { - - warning - +
+ @if (argumentFormGroup.get('argumentName').touched) { + @if (argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } } - } +
@@ -117,40 +119,42 @@ } - @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { -
-
{{ 'calculated-fields.timeseries-key' | translate }}
- -
- } @else { -
-
{{ 'calculated-fields.attribute-scope' | translate }}
- - - - {{ 'calculated-fields.server-attributes' | translate }} - - @if ((keyEntityType$ | async) === EntityType.DEVICE) { - - {{ 'calculated-fields.client-attributes' | translate }} - - - {{ 'calculated-fields.shared-attributes' | translate }} + @if (entityFilter.singleEntity.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { + @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { +
+
{{ 'calculated-fields.timeseries-key' | translate }}
+ +
+ } @else { +
+
{{ 'calculated-fields.attribute-scope' | translate }}
+ + + + {{ 'calculated-fields.server-attributes' | translate }} - } - - -
-
-
{{ 'calculated-fields.attribute-key' | translate }}
- -
+ @if ((keyEntityType$ | async) === EntityType.DEVICE) { + + {{ 'calculated-fields.client-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + } + + +
+
+
{{ 'calculated-fields.attribute-key' | translate }}
+ +
+ } } @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss index a784909b92..520f2d3ec0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -25,5 +25,9 @@ width: 100%; } } + + .mat-mdc-form-field-infix { + display: flex; + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index b476bc0fee..79f34d7c20 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, ElementRef, Input, OnInit, output, ViewChild } from '@angular/core'; +import { ChangeDetectorRef, Component, ElementRef, Input, OnInit, output, ViewChild } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; @@ -27,7 +27,7 @@ import { CalculatedFieldArgumentValue, CalculatedFieldType } from '@shared/models/calculated-field.models'; -import { debounceTime, delay, distinctUntilChanged, filter, map, startWith } from 'rxjs/operators'; +import { delay, distinctUntilChanged, filter, map, startWith, throttleTime } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; import { DatasourceType } from '@shared/models/widget.models'; @@ -92,6 +92,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme constructor( private fb: FormBuilder, + private cd: ChangeDetectorRef ) { super(); @@ -154,22 +155,24 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme default: entityId = this.argumentFormGroup.get('refEntityId').value as any; } - if (onInit) { + if (!onInit) { this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); } this.entityFilter = { type: AliasFilterType.singleEntity, singleEntity: entityId, }; + this.cd.markForCheck(); } private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, + this.refEntityKeyFormGroup.get('type').valueChanges, this.refEntityIdFormGroup.get('id').valueChanges.pipe(filter(Boolean)), this.refEntityKeyFormGroup.get('scope').valueChanges, ) - .pipe(debounceTime(300), delay(50), takeUntilDestroyed()) + .pipe(throttleTime(100), delay(50), takeUntilDestroyed()) .subscribe(() => this.updateEntityFilter(this.entityType)); } diff --git a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html index 44a218b455..cbec73e64d 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html @@ -29,7 +29,7 @@ {{ displayEntityFn(selectEntityFormGroup.get('entity').value) }} - - close + } @else if (keyControl.hasError('required') && keyControl.touched) { + + warning + } @for (key of filteredKeys$ | async; track key) { + } @empty { + {{ 'entity.no-keys-found' | translate }} } From 5f793b11f91d097187565810a63ac1856672e0b4 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 31 Jan 2025 16:06:52 +0200 Subject: [PATCH 152/416] BaseTelemetryProcessor - use async instead of .get() --- .../telemetry/BaseTelemetryProcessor.java | 161 +++++++++++------- 1 file changed, 95 insertions(+), 66 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java index 520c7ef5ff..5e7731db1d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import com.google.gson.Gson; import com.google.gson.JsonObject; @@ -81,6 +82,7 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -262,27 +264,38 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { return new ImmutablePair<>(queueName, ruleChainId); } - private ListenableFuture processPostAttributes(TenantId tenantId, CustomerId customerId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData, long ts) throws Exception { + private ListenableFuture processPostAttributes(TenantId tenantId, CustomerId customerId, EntityId entityId, + TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData, long ts) throws Exception { SettableFuture futureToSet = SettableFuture.create(); JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); - List attributes = new ArrayList<>(JsonConverter.convertToAttributes(json, ts)); - filterAttributesByTs(tenantId, entityId, AttributeScope.CLIENT_SCOPE, attributes, json); - - var defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId); - TbMsg tbMsg = TbMsg.newMsg() - .queueName(defaultQueueAndRuleChain.getKey()) - .type(TbMsgType.POST_ATTRIBUTES_REQUEST) - .originator(entityId) - .customerId(customerId) - .copyMetaData(metaData) - .data(gson.toJson(json)) - .ruleChainId(defaultQueueAndRuleChain.getValue()) - .build(); - edgeCtx.getClusterService().pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() { + ListenableFuture> future = filterAttributesByTs(tenantId, entityId, AttributeScope.CLIENT_SCOPE, attributes); + Futures.addCallback(future, new FutureCallback<>() { @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - futureToSet.set(null); + public void onSuccess(List attributesToSave) { + JsonObject jsonToSave = filterAttributesFromJson(json, attributesToSave); + var defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId); + TbMsg tbMsg = TbMsg.newMsg() + .queueName(defaultQueueAndRuleChain.getKey()) + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(entityId) + .customerId(customerId) + .copyMetaData(metaData) + .data(gson.toJson(jsonToSave)) + .ruleChainId(defaultQueueAndRuleChain.getValue()) + .build(); + edgeCtx.getClusterService().pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + futureToSet.set(null); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}] Can't process post attributes [{}]", tenantId, msg, t); + futureToSet.setException(t); + } + }); } @Override @@ -290,7 +303,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { log.error("[{}] Can't process post attributes [{}]", tenantId, msg, t); futureToSet.setException(t); } - }); + }, dbCallbackExecutorService); return futureToSet; } @@ -299,34 +312,46 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData, - long ts) throws Exception { + long ts) { SettableFuture futureToSet = SettableFuture.create(); JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); AttributeScope scope = AttributeScope.valueOf(metaData.getValue("scope")); List attributes = new ArrayList<>(JsonConverter.convertToAttributes(json, ts)); - List attributesToSave = filterAttributesByTs(tenantId, entityId, scope, attributes, json); - tsSubService.saveAttributes(AttributesSaveRequest.builder() - .tenantId(tenantId) - .entityId(entityId) - .scope(scope) - .entries(attributesToSave) - .callback(new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Void tmp) { - var defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId); - TbMsg tbMsg = TbMsg.newMsg() - .queueName(defaultQueueAndRuleChain.getKey()) - .type(TbMsgType.ATTRIBUTES_UPDATED) - .originator(entityId) - .customerId(customerId) - .copyMetaData(metaData) - .data(gson.toJson(json)) - .ruleChainId(defaultQueueAndRuleChain.getValue()) - .build(); - edgeCtx.getClusterService().pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() { + ListenableFuture> future = filterAttributesByTs(tenantId, entityId, scope, attributes); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(List attributesToSave) { + JsonObject jsonToSave = filterAttributesFromJson(json, attributesToSave); + tsSubService.saveAttributes(AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(entityId) + .scope(scope) + .entries(attributesToSave) + .callback(new FutureCallback<>() { @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - futureToSet.set(null); + public void onSuccess(@Nullable Void tmp) { + var defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId); + TbMsg tbMsg = TbMsg.newMsg() + .queueName(defaultQueueAndRuleChain.getKey()) + .type(TbMsgType.ATTRIBUTES_UPDATED) + .originator(entityId) + .customerId(customerId) + .copyMetaData(metaData) + .data(gson.toJson(jsonToSave)) + .ruleChainId(defaultQueueAndRuleChain.getValue()) + .build(); + edgeCtx.getClusterService().pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + futureToSet.set(null); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}] Can't process attributes update [{}]", tenantId, msg, t); + futureToSet.setException(t); + } + }); } @Override @@ -334,22 +359,28 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { log.error("[{}] Can't process attributes update [{}]", tenantId, msg, t); futureToSet.setException(t); } - }); - } + }) + .build()); + } - @Override - public void onFailure(Throwable t) { - log.error("[{}] Can't process attributes update [{}]", tenantId, msg, t); - futureToSet.setException(t); - } - }) - .build()); + @Override + public void onFailure(Throwable t) { + log.error("[{}] Can't process attributes update [{}]", tenantId, msg, t); + futureToSet.setException(t); + } + }, dbCallbackExecutorService); return futureToSet; } - private ListenableFuture processAttributeDeleteMsg(TenantId tenantId, EntityId entityId, AttributeDeleteMsg attributeDeleteMsg, - String entityType) { + private JsonObject filterAttributesFromJson(JsonObject json, List attributesToSave) { + Set keysToSave = attributesToSave.stream() + .map(KvEntry::getKey) + .collect(Collectors.toSet()); + json.keySet().removeIf(key -> !keysToSave.contains(key)); + return json; + } + private ListenableFuture processAttributeDeleteMsg(TenantId tenantId, EntityId entityId, AttributeDeleteMsg attributeDeleteMsg, String entityType) { String scope = attributeDeleteMsg.getScope(); List attributeKeys = attributeDeleteMsg.getAttributeNamesList(); ListenableFuture> removeAllFuture = edgeCtx.getAttributesService().removeAll(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys); @@ -401,21 +432,19 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { entityDataMsgConstructor.constructEntityDataMsg(tenantId, entityId, actionType, JsonParser.parseString(bodyJackson)); } - private List filterAttributesByTs(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes, JsonObject jsonObject) throws Exception { + private ListenableFuture> filterAttributesByTs(TenantId tenantId, EntityId entityId, AttributeScope scope, + List attributes) { List keys = attributes.stream().map(KvEntry::getKey).toList(); - Map existingAttributesTs = edgeCtx.getAttributesService().find(tenantId, entityId, scope, keys).get() - .stream().collect(Collectors.toMap(KvEntry::getKey, AttributeKvEntry::getLastUpdateTs)); - return attributes.stream() - .filter(attribute -> { - String key = attribute.getKey(); - long incomingTs = attribute.getLastUpdateTs(); - if (incomingTs > existingAttributesTs.getOrDefault(key, 0L)) { - return true; - } else { - jsonObject.remove(key); - return false; - } - }).toList(); + ListenableFuture> future = edgeCtx.getAttributesService().find(tenantId, entityId, scope, keys); + return Futures.transform(future, input -> { + Map existingAttributesTs = input.stream().collect(Collectors.toMap(KvEntry::getKey, AttributeKvEntry::getLastUpdateTs)); + return attributes.stream() + .filter(attribute -> { + String key = attribute.getKey(); + long incomingTs = attribute.getLastUpdateTs(); + return incomingTs > existingAttributesTs.getOrDefault(key, 0L); + }).toList(); + }, dbCallbackExecutorService); } } From 75369604a1cc1410b9af6249a3eb448d162a416a Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 16:46:28 +0200 Subject: [PATCH 153/416] Fixes --- .../calculated-field-arguments-table.component.html | 2 +- .../calculated-field-arguments-table.component.ts | 12 ++++++++---- .../dialog/calculated-field-dialog.component.ts | 4 +++- .../calculated-field-argument-panel.component.html | 3 ++- .../calculated-field-argument-panel.component.ts | 13 ++++--------- ui-ngx/src/app/shared/models/regex.constants.ts | 2 ++ ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 7 files changed, 21 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 021bd15c58..f6b82dd954 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -99,7 +99,7 @@ {{ 'calculated-fields.no-arguments' | translate }} } - @if (errorText) { + @if (errorText && this.argumentsFormArray.dirty) { } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 657b20abb4..7357912bdc 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -50,7 +50,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { isDefinedAndNotNull } from '@core/utils'; -import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { charNumRegex } from '@shared/models/regex.constants'; @Component({ selector: 'tb-calculated-field-arguments-table', @@ -98,7 +98,11 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { this.propagateChange(this.getArgumentsObject()); }); - effect(() => this.calculatedFieldType() && this.argumentsFormArray.updateValueAndValidity()); + effect(() => { + if (this.calculatedFieldType() && this.argumentsFormArray.dirty) { + this.argumentsFormArray.updateValueAndValidity(); + } + }); } registerOnChange(fn: (argumentsObj: Record) => void): void { @@ -183,7 +187,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], ...argumentsObj[key], ...(argumentsObj[key].refEntityId ? { refEntityId: this.fb.group({ @@ -202,7 +206,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { return this.fb.group({ ...value, - argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], ...(value.refEntityId ? { refEntityId: this.fb.group({ entityType: [{ value: value.refEntityId.entityType, disabled: true }], diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 71fea36959..d30a90e954 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -110,7 +110,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent {{ 'calculated-fields.server-attributes' | translate }} - @if ((keyEntityType$ | async) === EntityType.DEVICE) { + @if (entityType === ArgumentEntityType.Device + || entityType === ArgumentEntityType.Current && entityId.entityType === EntityType.DEVICE) { {{ 'calculated-fields.client-attributes' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 79f34d7c20..2949ffdaee 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -18,7 +18,7 @@ import { ChangeDetectorRef, Component, ElementRef, Input, OnInit, output, ViewCh import { TbPopoverComponent } from '@shared/components/popover.component'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; -import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { charNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, ArgumentEntityTypeTranslations, @@ -27,7 +27,7 @@ import { CalculatedFieldArgumentValue, CalculatedFieldType } from '@shared/models/calculated-field.models'; -import { delay, distinctUntilChanged, filter, map, startWith, throttleTime } from 'rxjs/operators'; +import { delay, distinctUntilChanged, filter, throttleTime } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; import { DatasourceType } from '@shared/models/widget.models'; @@ -57,7 +57,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, Validators.pattern(charNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -74,11 +74,6 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme argumentTypes: ArgumentType[]; entityFilter: EntityFilter; - keyEntityType$ = this.refEntityIdFormGroup.get('entityType').valueChanges - .pipe( - startWith(this.refEntityIdFormGroup.get('entityType').value), - map(type => type === ArgumentEntityType.Current ? this.entityId.entityType : type) - ); readonly argumentEntityTypes = Object.values(ArgumentEntityType) as ArgumentEntityType[]; readonly ArgumentEntityTypeTranslations = ArgumentEntityTypeTranslations; @@ -140,7 +135,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme this.argumentFormGroup.get('defaultValue')[isRolling? 'disable' : 'enable']({ emitEvent: false }); } - private updateEntityFilter(entityType: ArgumentEntityType, onInit = false): void { + private updateEntityFilter(entityType: ArgumentEntityType = ArgumentEntityType.Current, onInit = false): void { let entityId: EntityId; switch (entityType) { case ArgumentEntityType.Current: diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts index 55742cefd4..60bf154423 100644 --- a/ui-ngx/src/app/shared/models/regex.constants.ts +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -15,3 +15,5 @@ /// export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; + +export const charNumRegex = /^[a-zA-Z0-9]+$/; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index be31b4f70b..ab696e226c 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1030,7 +1030,7 @@ "timeseries-key": "Time series key", "device-name": "Device name", "latest-telemetry": "Latest telemetry", - "rolling": "Rolling", + "rolling": "Time series rolling", "attribute-scope": "Attribute scope", "server-attributes": "Server attributes", "client-attributes": "Client attributes", From ff8a9309f4a0cba877aa5d401a325021ed1d2cf6 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 16:57:46 +0200 Subject: [PATCH 154/416] Fixes --- .../calculated-field-arguments-table.component.html | 4 ++-- .../panel/calculated-field-argument-panel.component.ts | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index f6b82dd954..4f1d1d3980 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -28,7 +28,7 @@ - @if (group.get('refEntityId')?.get('id').value) { + @if (group.get('refEntityId')?.get('id')?.value) { @@ -51,7 +51,7 @@ {{ - (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant + (group.get('refEntityId')?.get('entityType')?.value === ArgumentEntityType.Tenant ? 'calculated-fields.argument-current-tenant' : 'calculated-fields.argument-current') | translate }} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 2949ffdaee..6e232bdb3d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -119,7 +119,9 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme } saveArgument(): void { - this.argumentsDataApplied.emit({ value: this.argumentFormGroup.value as CalculatedFieldArgumentValue, index: this.index }); + const { refEntityId, ...restConfig } = this.argumentFormGroup.value; + const value = (refEntityId.entityType === ArgumentEntityType.Current ? restConfig : { refEntityId, ...restConfig }) as CalculatedFieldArgumentValue; + this.argumentsDataApplied.emit({ value, index: this.index }); } cancel(): void { @@ -148,7 +150,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme }; break; default: - entityId = this.argumentFormGroup.get('refEntityId').value as any; + entityId = this.argumentFormGroup.get('refEntityId').value as unknown as EntityId; } if (!onInit) { this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); From ce4c654e21d89784a7b893f1abe9bac56ecb1e21 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 31 Jan 2025 16:58:12 +0200 Subject: [PATCH 155/416] BaseTelemetryProcessor - use scope from msg, and not hardcoded CLIENT_SCOPE --- .../processor/telemetry/BaseTelemetryProcessor.java | 8 +++++--- .../rule/engine/edge/AbstractTbMsgPushNode.java | 10 ++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java index 5e7731db1d..73713ccf99 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java @@ -130,11 +130,12 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { CustomerId customerId = pair.getValue(); metaData.putValue(DataConstants.MSG_SOURCE_KEY, getMsgSourceKey()); if (entityData.hasPostAttributesMsg()) { + metaData.putValue(DataConstants.SCOPE, entityData.getPostAttributeScope()); long ts = entityData.hasAttributeTs() ? entityData.getAttributeTs() : System.currentTimeMillis(); result.add(processPostAttributes(tenantId, customerId, entityId, entityData.getPostAttributesMsg(), metaData, ts)); } if (entityData.hasAttributesUpdatedMsg()) { - metaData.putValue("scope", entityData.getPostAttributeScope()); + metaData.putValue(DataConstants.SCOPE, entityData.getPostAttributeScope()); long ts = entityData.hasAttributeTs() ? entityData.getAttributeTs() : System.currentTimeMillis(); result.add(processAttributesUpdate(tenantId, customerId, entityId, entityData.getAttributesUpdatedMsg(), metaData, ts)); } @@ -268,8 +269,9 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData, long ts) throws Exception { SettableFuture futureToSet = SettableFuture.create(); JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); + AttributeScope scope = AttributeScope.valueOf(metaData.getValue(DataConstants.SCOPE)); List attributes = new ArrayList<>(JsonConverter.convertToAttributes(json, ts)); - ListenableFuture> future = filterAttributesByTs(tenantId, entityId, AttributeScope.CLIENT_SCOPE, attributes); + ListenableFuture> future = filterAttributesByTs(tenantId, entityId, scope, attributes); Futures.addCallback(future, new FutureCallback<>() { @Override public void onSuccess(List attributesToSave) { @@ -315,7 +317,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { long ts) { SettableFuture futureToSet = SettableFuture.create(); JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); - AttributeScope scope = AttributeScope.valueOf(metaData.getValue("scope")); + AttributeScope scope = AttributeScope.valueOf(metaData.getValue(DataConstants.SCOPE)); List attributes = new ArrayList<>(JsonConverter.convertToAttributes(json, ts)); ListenableFuture> future = filterAttributesByTs(tenantId, entityId, scope, attributes); Futures.addCallback(future, new FutureCallback<>() { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java index f1fe1affc6..911bfac43f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java @@ -54,8 +54,6 @@ public abstract class AbstractTbMsgPushNode { entityBody.put("kv", dataJson); entityBody.put("ts", msg.getMetaDataTs()); - entityBody.put(SCOPE, getScope(metadata)); + entityBody.put(DataConstants.SCOPE, getScope(metadata)); if (EdgeEventActionType.POST_ATTRIBUTES.equals(actionType)) { entityBody.put("isPostAttributes", true); } @@ -99,7 +97,7 @@ public abstract class AbstractTbMsgPushNode keys = JacksonUtil.convertValue(dataJson.get("attributes"), new TypeReference<>() { }); entityBody.put("keys", keys); - entityBody.put(SCOPE, getScope(metadata)); + entityBody.put(DataConstants.SCOPE, getScope(metadata)); } case TIMESERIES_UPDATED -> { entityBody.put("data", dataJson); @@ -146,7 +144,7 @@ public abstract class AbstractTbMsgPushNode metadata) { - String scope = metadata.get(SCOPE); + String scope = metadata.get(DataConstants.SCOPE); if (StringUtils.isEmpty(scope)) { scope = config.getScope(); } @@ -164,7 +162,7 @@ public abstract class AbstractTbMsgPushNode Date: Fri, 31 Jan 2025 17:02:07 +0200 Subject: [PATCH 156/416] BaseTelemetryProcessor - Fixed import --- .../edge/rpc/processor/telemetry/BaseTelemetryProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java index 73713ccf99..4251ee8a79 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import com.google.gson.Gson; import com.google.gson.JsonObject; From 6905cb530b915898cf90201bde255a3507c20293 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Jan 2025 17:12:02 +0200 Subject: [PATCH 157/416] added new protos for states --- ...efaultCalculatedFieldExecutionService.java | 23 +-- .../server/service/cf/RocksDBService.java | 28 +++- .../cf/ctx/CalculatedFieldStateService.java | 2 - .../cf/ctx/state/RocksDBStateService.java | 149 ++++++++++++++++-- .../ctx/state/ScriptCalculatedFieldState.java | 3 +- .../ctx/state/SimpleCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 14 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 28 +++- .../ctx/state/TsRollingArgumentEntryTest.java | 25 +-- .../server/common/util/ProtoUtils.java | 39 ++++- common/proto/src/main/proto/queue.proto | 25 ++- 11 files changed, 263 insertions(+), 77 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 8f948aa265..7367eb611f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -71,11 +71,13 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -84,7 +86,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -383,7 +384,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> entry.getValue().getRefEntityId() != null) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); @@ -551,7 +552,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + .collect(Collectors.toMap(Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); // updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); } @@ -677,7 +678,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (broadcast) { broadcasts.add(link); } else { - TopicPartitionInfo tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, link.entityId()); + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, link.entityId()); unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); } } @@ -710,7 +711,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { - TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + Builder builder = CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); builder.setMsg(telemetryProto); for (CalculatedFieldEntityCtxId link : links) { builder.addLinks(toProto(link)); @@ -719,8 +720,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } //TODO: IM: move to utils; - private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) .setEntityType(ctxId.entityId().getEntityType().name()) @@ -890,8 +893,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return telemetryMsg; } - private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { - return TransportProtos.CalculatedFieldIdProto.newBuilder() + private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) .build(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index 3aed65eced..5eaeff120a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -22,6 +22,8 @@ import org.rocksdb.RocksIterator; import org.rocksdb.WriteOptions; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.utils.RocksDBConfig; import java.nio.charset.StandardCharsets; @@ -49,6 +51,14 @@ public class RocksDBService { } } + public void put(CalculatedFieldEntityCtxIdProto key, CalculatedFieldStateProto value) { + try { + db.put(writeOptions, key.toByteArray(), value.toByteArray()); + } catch (RocksDBException e) { + log.error("Failed to store data to RocksDB", e); + } + } + public void delete(String key) { try { db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); @@ -67,18 +77,20 @@ public class RocksDBService { } } - public Map getAll() { - Map map = new HashMap<>(); + public Map getAll() { + Map results = new HashMap<>(); try (RocksIterator iterator = db.newIterator()) { for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - String key = new String(iterator.key(), StandardCharsets.UTF_8); - String value = new String(iterator.value(), StandardCharsets.UTF_8); - map.put(key, value); + try { + CalculatedFieldEntityCtxIdProto key = CalculatedFieldEntityCtxIdProto.parseFrom(iterator.key()); + CalculatedFieldStateProto value = CalculatedFieldStateProto.parseFrom(iterator.value()); + results.put(key, value); + } catch (Exception e) { + log.error("Failed to retrieve data from RocksDB", e); + } } - } catch (Exception e) { - log.error("Failed to retrieve data from RocksDB", e); } - return map; + return results; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index 2bf6b7e0f2..4f9a998937 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -24,8 +24,6 @@ public interface CalculatedFieldStateService { Map restoreStates(); - CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId); - void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId ctxId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index aa82f2fc57..4de15f70bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -19,14 +19,28 @@ import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.RollingArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueProto; import org.thingsboard.server.service.cf.RocksDBService; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import java.util.Map; -import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; import java.util.stream.Collectors; @Service @@ -40,21 +54,14 @@ public class RocksDBStateService implements CalculatedFieldStateService { public Map restoreStates() { return rocksDBService.getAll().entrySet().stream() .collect(Collectors.toMap( - entry -> JacksonUtil.fromString(entry.getKey(), CalculatedFieldEntityCtxId.class), - entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldState.class) + entry -> fromProto(entry.getKey()), + entry -> fromProto(entry.getValue()) )); } @Override - public CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId) { - return Optional.ofNullable(rocksDBService.get(JacksonUtil.writeValueAsString(ctxId))) - .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldState.class)) - .orElse(null); - } - - @Override - public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback){ - rocksDBService.put(JacksonUtil.writeValueAsString(stateId), JacksonUtil.writeValueAsString(state)); + public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + rocksDBService.put(toProto(stateId), toProto(stateId, state)); callback.onSuccess(); } @@ -63,4 +70,120 @@ public class RocksDBStateService implements CalculatedFieldStateService { rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } + private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + private CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { + TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + private CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { + CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() + .setId(toProto(stateId)) + .setType(state.getType().name()) + .addAllRequiredArguments(state.getRequiredArguments()); + + state.getArguments().forEach((argName, argEntry) -> { + if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { + builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); + } + }); + + return builder.build(); + } + + private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { + SingleValueProto.Builder singleValueProtoBuilder = SingleValueProto.newBuilder() + .setTs(entry.getTs()); + + if (entry.getVersion() != null) { + singleValueProtoBuilder.setVersion(entry.getVersion()); + } + + KvEntry value = entry.getValue(); + if (value != null) { + singleValueProtoBuilder.setHasV(true) + .setValue(ProtoUtils.toKeyValueProto(value)); + } + + return SingleValueArgumentProto.newBuilder() + .setArgName(argName) + .setValue(singleValueProtoBuilder.build()) + .build(); + } + + private RollingArgumentProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { + RollingArgumentProto.Builder rollingArgumentProtoBuilder = RollingArgumentProto.newBuilder() + .setArgName(argName); + + entry.getTsRecords().forEach((ts, value) -> { + SingleValueProto.Builder singleValueProtoBuilder = SingleValueProto.newBuilder() + .setTs(ts); + + if (value != null) { + singleValueProtoBuilder.setHasV(true) + .setValue(ProtoUtils.toKeyValueProto(value)); + } + + rollingArgumentProtoBuilder.addValues(singleValueProtoBuilder.build()); + }); + + return rollingArgumentProtoBuilder.build(); + } + + private CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { + if (StringUtils.isEmpty(proto.getType())) { + return null; + } + + CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); + + CalculatedFieldState state = switch (type) { + case SIMPLE -> new SimpleCalculatedFieldState(proto.getRequiredArgumentsList()); + case SCRIPT -> new ScriptCalculatedFieldState(proto.getRequiredArgumentsList()); + }; + + proto.getSingleValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); + + if (CalculatedFieldType.SCRIPT.equals(type)) { + proto.getRollingValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromRollingArgumentProto(argProto))); + } + + return state; + } + + private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { + SingleValueProto valueProto = proto.getValue(); + BasicKvEntry value = valueProto.getHasV() ? ProtoUtils.fromProto(valueProto.getValue()) : null; + + return new SingleValueArgumentEntry(valueProto.getTs(), value, valueProto.getVersion()); + } + + private TsRollingArgumentEntry fromRollingArgumentProto(RollingArgumentProto proto) { + TreeMap tsRecords = new TreeMap<>(); + + proto.getValuesList().forEach(singleValueProto -> { + BasicKvEntry value = singleValueProto.getHasV() ? ProtoUtils.fromProto(singleValueProto.getValue()) : null; + tsRecords.put(singleValueProto.getTs(), value); + }); + + return new TsRollingArgumentEntry(tsRecords); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index a3c7efb388..4a24d13c93 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.List; @@ -53,7 +54,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { arguments.forEach((key, argumentEntry) -> { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingEntry) { Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = tsRollingEntry.getTsRecords(); + TreeMap tsRecords = tsRollingEntry.getTsRecords(); if (tsRecords.size() > argument.getLimit()) { tsRecords.pollFirstEntry(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 8b8fe6e8c7..d233b60512 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -21,6 +21,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.List; @@ -52,7 +53,8 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { for (Map.Entry entry : this.arguments.entrySet()) { try { - expr.setVariable(entry.getKey(), Double.parseDouble(entry.getValue().getValue().toString())); + BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getValue(); + expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); } catch (NumberFormatException e) { throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 5117fcab0e..8d4d40d39b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.util.ProtoUtils; @@ -33,19 +34,19 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); private long ts; - private Object value; + private BasicKvEntry value; private Long version; public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.fromProto(entry).getValue(); + this.value = ProtoUtils.fromProto(entry.getKv()); } public SingleValueArgumentEntry(AttributeValueProto entry) { this.ts = entry.getLastUpdateTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.fromProto(entry).getValue(); + this.value = ProtoUtils.basicKvEntryFromProto(entry); } public SingleValueArgumentEntry(KvEntry entry) { @@ -56,7 +57,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.ts = attributeKvEntry.getLastUpdateTs(); this.version = attributeKvEntry.getVersion(); } - this.value = entry.getValue(); + this.value = ProtoUtils.basicKvEntryFromKvEntry(entry); } /** @@ -72,11 +73,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return ArgumentEntryType.SINGLE_VALUE; } - @Override - public Object getValue() { - return value; - } - @Override public ArgumentEntry copy() { return new SingleValueArgumentEntry(this.ts, this.value, this.version); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index b86a51ca03..6c1a772c44 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -21,11 +21,15 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.util.ProtoUtils; import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.stream.Collectors; @Data @NoArgsConstructor @@ -37,10 +41,10 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; - private TreeMap tsRecords = new TreeMap<>(); + private TreeMap tsRecords = new TreeMap<>(); public TsRollingArgumentEntry(List kvEntries) { - kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry.getValue())); + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); } /** @@ -58,7 +62,15 @@ public class TsRollingArgumentEntry implements ArgumentEntry { @JsonIgnore @Override public Object getValue() { - return tsRecords; + return tsRecords.entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue().getValue(), + (oldValue, newValue) -> oldValue, + TreeMap::new + )); + } @Override @@ -79,7 +91,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private boolean updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { boolean updated = false; - for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { + for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { updated |= addTsRecordIfAbsent(tsRecordEntry.getKey(), tsRecordEntry.getValue()); } return updated; @@ -89,7 +101,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getValue()); } - private boolean addTsRecordIfAbsent(Long ts, Object value) { + private boolean addTsRecordIfAbsent(Long ts, KvEntry value) { if (!tsRecords.containsKey(ts)) { addTsRecord(ts, value); return true; @@ -97,9 +109,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return false; } - private void addTsRecord(Long ts, Object value) { - if (NumberUtils.isParsable(value.toString())) { - tsRecords.put(ts, value); + private void addTsRecord(Long ts, KvEntry value) { + if (NumberUtils.isParsable(value.getValue().toString())) { + tsRecords.put(ts, ProtoUtils.basicKvEntryFromKvEntry(value)); if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { tsRecords.pollFirstEntry(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index b08c5f2a58..9ca242092d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -17,6 +17,9 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import java.util.Map; import java.util.TreeMap; @@ -32,10 +35,10 @@ public class TsRollingArgumentEntryTest { @BeforeEach void setUp() { - TreeMap values = new TreeMap<>(); - values.put(ts - 40, 10); - values.put(ts - 30, 12); - values.put(ts - 20, 17); + TreeMap values = new TreeMap<>(); + values.put(ts - 40, new DoubleDataEntry("key", 10.0)); + values.put(ts - 30, new DoubleDataEntry("key", 12.0)); + values.put(ts - 20, new DoubleDataEntry("key", 17.0)); entry = new TsRollingArgumentEntry(values); } @@ -47,7 +50,7 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenSingleValueEntryPassed() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, 23, 123L); + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new DoubleDataEntry("key", 23.0), 123L); assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(4); @@ -56,7 +59,7 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, 23, 123L); + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, new DoubleDataEntry("key", 23.0), 123L); assertThat(entry.updateEntry(newEntry)).isFalse(); } @@ -64,10 +67,10 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenRollingEntryPassed() { TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 20, 16); - values.put(ts - 10, 7); - values.put(ts - 5, 1); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, new DoubleDataEntry("key", 16.0)); + values.put(ts - 10, new DoubleDataEntry("key", 7.0)); + values.put(ts - 5, new DoubleDataEntry("key", 1.0)); newEntry.setTsRecords(values); assertThat(entry.updateEntry(newEntry)).isTrue(); @@ -83,7 +86,7 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenValueIsNotNumber() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, "string", 123L); + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new StringDataEntry("key", "string"), 123L); assertThatThrownBy(() -> entry.updateEntry(newEntry)) .isInstanceOf(IllegalArgumentException.class) diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 7ba11d8ea9..91e3276210 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -58,6 +58,7 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; @@ -90,7 +91,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import java.util.ArrayList; import java.util.Arrays; @@ -630,6 +631,42 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static BasicKvEntry basicKvEntryFromProto(TransportProtos.AttributeValueProto proto) { + boolean hasValue = proto.getHasV(); + String key = proto.getKey(); + return switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, hasValue ? proto.getBoolV() : null); + case LONG_V -> new LongDataEntry(key, hasValue ? proto.getLongV() : null); + case DOUBLE_V -> new DoubleDataEntry(key, hasValue ? proto.getDoubleV() : null); + case STRING_V -> new StringDataEntry(key, hasValue ? proto.getStringV() : null); + case JSON_V -> new JsonDataEntry(key, hasValue ? proto.getJsonV() : null); + default -> null; + }; + } + + public static BasicKvEntry fromProto(KeyValueProto proto) { + String key = proto.getKey(); + return switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, proto.getBoolV()); + case LONG_V -> new LongDataEntry(key, proto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, proto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, proto.getStringV()); + case JSON_V -> new JsonDataEntry(key, proto.getJsonV()); + default -> null; + }; + } + + public static BasicKvEntry basicKvEntryFromKvEntry(KvEntry kvEntry) { + String key = kvEntry.getKey(); + return switch (kvEntry.getDataType()) { + case BOOLEAN -> new BooleanDataEntry(key, kvEntry.getBooleanValue().orElse(null)); + case LONG -> new LongDataEntry(key, kvEntry.getLongValue().orElse(null)); + case DOUBLE -> new DoubleDataEntry(key, kvEntry.getDoubleValue().orElse(null)); + case STRING -> new StringDataEntry(key, kvEntry.getStrValue().orElse(null)); + case JSON -> new JsonDataEntry(key, kvEntry.getJsonValue().orElse(null)); + }; + } + public static TsKvEntry fromProto(TransportProtos.TsKvProto proto) { TransportProtos.KeyValueProto kvProto = proto.getKv(); String key = kvProto.getKey(); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 53045fc147..2ed535495a 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -810,11 +810,13 @@ message CalculatedFieldLinkedTelemetryMsgProto { } message CalculatedFieldEntityCtxIdProto { - int64 calculatedFieldIdMSB = 1; - int64 calculatedFieldIdLSB = 2; - string entityType = 3; - int64 entityIdMSB = 4; - int64 entityIdLSB = 5; + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + string entityType = 5; + int64 entityIdMSB = 6; + int64 entityIdLSB = 7; } message CalculatedFieldIdProto { @@ -825,13 +827,8 @@ message CalculatedFieldIdProto { message SingleValueProto { int64 ts = 1; int64 version = 2; - KeyValueType type = 3; bool has_v = 4; - bool bool_v = 5; - int64 long_v = 6; - double double_v = 7; - string string_v = 8; - string json_v = 9; + KeyValueProto value = 5; } message SingleValueArgumentProto { @@ -847,8 +844,10 @@ message RollingArgumentProto { message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; // int32 version = 2; - repeated SingleValueArgumentProto singleValueArguments = 3; - repeated RollingArgumentProto rollingValueArguments = 4; + string type = 3; + repeated string requiredArguments = 4; + repeated SingleValueArgumentProto singleValueArguments = 5; + repeated RollingArgumentProto rollingValueArguments = 6; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From 99effb6918b897f9d502e4272e286f823b2e48b4 Mon Sep 17 00:00:00 2001 From: Yevhenii Date: Fri, 31 Jan 2025 18:00:24 +0200 Subject: [PATCH 158/416] ERRORs processing RESOURCES - Disabled duplicate validation on cloud for resource message --- .../edge/rpc/processor/resource/BaseResourceProcessor.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java index ac7bd6768c..4608e52cbf 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java @@ -17,24 +17,18 @@ package org.thingsboard.server.service.edge.rpc.processor.resource; import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.id.TbResourceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j public abstract class BaseResourceProcessor extends BaseEdgeProcessor { - @Autowired - private DataValidator resourceValidator; - protected boolean saveOrUpdateTbResource(TenantId tenantId, TbResourceId tbResourceId, ResourceUpdateMsg resourceUpdateMsg) { boolean resourceKeyUpdated = false; try { @@ -64,7 +58,6 @@ public abstract class BaseResourceProcessor extends BaseEdgeProcessor { } } resource.setResourceKey(resourceKey); - resourceValidator.validate(resource, TbResourceInfo::getTenantId); if (created) { resource.setId(tbResourceId); } From 14feedbfa05b11f1d20dfa0f6fe9c17a7ea03f57 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 18:01:28 +0200 Subject: [PATCH 159/416] Changed UI of timeWindow --- ...ulated-field-argument-panel.component.html | 22 +++++++++---------- ...lculated-field-argument-panel.component.ts | 5 +++-- .../assets/locale/locale.constant-en_US.json | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 7c33cc2eb3..b1d463a2d3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -168,22 +168,20 @@ } @else { -
-
{{ 'calculated-fields.time-window' | translate }}
-
- - - {{ 'common.suffix.ms' | translate }} - +
+
{{ 'calculated-fields.time-window' | translate }}
+
+
{{ 'calculated-fields.limit' | translate }}
-
- - - -
+
}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 6e232bdb3d..354b6b9f58 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -36,6 +36,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityFilter } from '@shared/models/query/query.models'; import { AliasFilterType } from '@shared/models/alias.models'; import { merge } from 'rxjs'; +import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', @@ -68,8 +69,8 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], }), defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], - limit: [null], - timeWindow: [null], + limit: [10], + timeWindow: [MINUTE * 15], }); argumentTypes: ArgumentType[]; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ab696e226c..f0f0671c5c 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1037,7 +1037,7 @@ "shared-attributes": "Shared attributes", "attribute-key": "Attribute key", "default-value": "Default value", - "limit": "Limit", + "limit": "Max values", "time-window": "Time window", "customer-name": "Customer name", "timeseries": "Time series", From 0f7142cfef7f04afc69ebf5f2a63885accc0b3f3 Mon Sep 17 00:00:00 2001 From: Yevhenii Date: Fri, 31 Jan 2025 18:11:51 +0200 Subject: [PATCH 160/416] ERRORs processing DEVICE PROFILE - Added saving the DELETED action when the edge is offline. - Added synchronization of Device Profile twice for its relation with Ota Package. --- .../org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java | 1 + .../server/service/edge/rpc/processor/BaseEdgeProcessor.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java index 7426207a56..7fab0094b4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java @@ -87,6 +87,7 @@ public class EdgeSyncCursor { fetchers.add(new SystemWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService())); fetchers.add(new TenantWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService())); fetchers.add(new OtaPackagesEdgeEventFetcher(ctx.getOtaPackageService())); + fetchers.add(new DeviceProfilesEdgeEventFetcher(ctx.getDeviceProfileService())); fetchers.add(new TenantResourcesEdgeEventFetcher(ctx.getResourceService())); fetchers.add(new OAuth2EdgeEventFetcher(ctx.getDomainService())); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index e448fbd937..8f0f06c6bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -120,7 +120,7 @@ public abstract class BaseEdgeProcessor { private boolean doSaveIfEdgeIsOffline(EdgeEventType type, EdgeEventActionType action) { return switch (action) { case TIMESERIES_UPDATED, ALARM_ACK, ALARM_CLEAR, ALARM_ASSIGNED, ALARM_UNASSIGNED, ADDED_COMMENT, - UPDATED_COMMENT -> true; + UPDATED_COMMENT, DELETED -> true; default -> switch (type) { case ALARM, ALARM_COMMENT, RULE_CHAIN, RULE_CHAIN_METADATA, USER, CUSTOMER, TENANT, TENANT_PROFILE, WIDGETS_BUNDLE, WIDGET_TYPE, ADMIN_SETTINGS, OTA_PACKAGE, QUEUE, RELATION, NOTIFICATION_TEMPLATE, NOTIFICATION_TARGET, From 3b8be75a14fe87968a5823346db4f1f8afcadf1c Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 31 Jan 2025 18:24:50 +0200 Subject: [PATCH 161/416] DeviceEdgeTest.testSendOutdatedAttributeToCloud - improved stability for concurrent issue --- .../java/org/thingsboard/server/edge/DeviceEdgeTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java index 060e641b93..80e7d2aed4 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java @@ -500,6 +500,15 @@ public class DeviceEdgeTest extends AbstractEdgeTest { attributesNode.put("test_attr", originalValue); doPost("/api/plugins/telemetry/DEVICE/" + device.getId() + "/attributes/SERVER_SCOPE", attributesNode); + // Wait before device attributes saved to database + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> { + String urlTemplate = "/api/plugins/telemetry/DEVICE/" + device.getId() + "/keys/attributes/" + DataConstants.SERVER_SCOPE; + List actualKeys = doGetAsyncTyped(urlTemplate, new TypeReference<>() {}); + return actualKeys != null && !actualKeys.isEmpty() && actualKeys.contains("test_attr"); + }); + JsonObject attributesData = new JsonObject(); // incorrect msg, will not be saved, because of ts is lower than for already existing String attributesKey = "test_attr"; From 8dca91c909138deb18a5b8d1b6aed332bcad2e2e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 19:08:56 +0200 Subject: [PATCH 162/416] Fixed device filter bug --- .../calculated-fields/calculated-fields-table-config.ts | 6 +++--- .../calculated-fields-table.component.html | 4 +++- .../calculated-fields/calculated-fields-table.component.ts | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index fcef11799d..58caa759da 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -97,7 +97,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, - onAction: (_, entity) => this.editCalculatedField(entity) + onAction: (_, entity) => this.editCalculatedField(entity), } ); } @@ -171,7 +171,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig - +@if (calculatedFieldsTableConfig) { + +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 4c22ad2896..4fda1cc075 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -16,6 +16,7 @@ import { ChangeDetectionStrategy, + ChangeDetectorRef, Component, DestroyRef, effect, @@ -43,7 +44,7 @@ export class CalculatedFieldsTableComponent { @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; - active = input(); + active = input(); entityId = input(); calculatedFieldsTableConfig: CalculatedFieldsTableConfig; @@ -54,6 +55,7 @@ export class CalculatedFieldsTableComponent { private store: Store, private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, + private cd: ChangeDetectorRef, private destroyRef: DestroyRef) { effect(() => { @@ -68,6 +70,7 @@ export class CalculatedFieldsTableComponent { this.popoverService, this.destroyRef, ); + this.cd.markForCheck(); } }); } From dd8ce35b86f2da4102bb1bfebceada167bc72a99 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 19:18:57 +0200 Subject: [PATCH 163/416] Fixes --- .../calculated-field-arguments-table.component.ts | 6 +++--- .../dialog/calculated-field-dialog.component.ts | 2 +- .../panel/calculated-field-argument-panel.component.html | 2 +- .../panel/calculated-field-argument-panel.component.ts | 8 ++++---- .../entity/entity-key-autocomplete.component.html | 4 +++- ui-ngx/src/app/shared/models/regex.constants.ts | 2 +- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 7357912bdc..c8dae67aec 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -50,7 +50,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { isDefinedAndNotNull } from '@core/utils'; -import { charNumRegex } from '@shared/models/regex.constants'; +import { charsWithNumRegex } from '@shared/models/regex.constants'; @Component({ selector: 'tb-calculated-field-arguments-table', @@ -187,7 +187,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], ...argumentsObj[key], ...(argumentsObj[key].refEntityId ? { refEntityId: this.fb.group({ @@ -206,7 +206,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { return this.fb.group({ ...value, - argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], + argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], ...(value.refEntityId ? { refEntityId: this.fb.group({ entityType: [{ value: value.refEntityId.entityType, disabled: true }], diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index d30a90e954..6d0687fe9a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -61,7 +61,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) ); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index b1d463a2d3..020fad4fd2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -23,7 +23,7 @@
{{ 'calculated-fields.argument-name' | translate }}
- +
@if (argumentFormGroup.get('argumentName').touched) { @if (argumentFormGroup.get('argumentName').hasError('required')) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 354b6b9f58..792742e5d0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -18,7 +18,7 @@ import { ChangeDetectorRef, Component, ElementRef, Input, OnInit, output, ViewCh import { TbPopoverComponent } from '@shared/components/popover.component'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; -import { charNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, ArgumentEntityTypeTranslations, @@ -58,7 +58,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, Validators.pattern(charNumRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -66,10 +66,10 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme refEntityKey: this.fb.group({ type: [ArgumentType.LatestTelemetry, [Validators.required]], key: [''], - scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }, [Validators.required]], }), defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], - limit: [10], + limit: [1000], timeWindow: [MINUTE * 15], }); diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html index 3e4ff6e90d..f1073738f9 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html @@ -43,7 +43,9 @@ @for (key of filteredKeys$ | async; track key) { } @empty { - {{ 'entity.no-keys-found' | translate }} + @if (!this.keyControl.value) { + {{ 'entity.no-keys-found' | translate }} + } } diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts index 60bf154423..c6b231ef6e 100644 --- a/ui-ngx/src/app/shared/models/regex.constants.ts +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -16,4 +16,4 @@ export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; -export const charNumRegex = /^[a-zA-Z0-9]+$/; +export const charsWithNumRegex = /^[a-zA-Z]+[a-zA-Z0-9]*$/; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index f0f0671c5c..ee348cb6c3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1048,7 +1048,7 @@ "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", "hint": { - "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with rolling type.", + "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-empty": "Arguments should not be empty.", "expression-required": "Expression is required.", "expression-invalid": "Expression is invalid", From c82b6c8c45291d35bb15b128d7313167dd8a9dda Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 19:20:26 +0200 Subject: [PATCH 164/416] Empty arguments fix --- .../components/dialog/calculated-field-dialog.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 6d0687fe9a..c9f1a22157 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -34,7 +34,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { map } from 'rxjs/operators'; +import { map, startWith } from 'rxjs/operators'; import { isObject } from '@core/utils'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -63,6 +63,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) ); From c3de38305f2ef8210c52f62729c5f21d8faf58d1 Mon Sep 17 00:00:00 2001 From: Illia Tsybulenko-Sihov Date: Fri, 31 Jan 2025 22:27:18 +0200 Subject: [PATCH 165/416] updated prometheus version to 3.1.0 --- docker/docker-compose.prometheus-grafana.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.prometheus-grafana.yml b/docker/docker-compose.prometheus-grafana.yml index 311d96bd86..f6a5f7fb6d 100644 --- a/docker/docker-compose.prometheus-grafana.yml +++ b/docker/docker-compose.prometheus-grafana.yml @@ -23,7 +23,7 @@ volumes: services: prometheus: - image: prom/prometheus:v2.1.0 + image: prom/prometheus:v3.1.0 volumes: - ./monitoring/prometheus/:/etc/prometheus/ - prometheus_data:/prometheus From 1a67769f1c3657bca7e1a8a31e558c7c0e5ac0e5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 3 Feb 2025 12:01:06 +0200 Subject: [PATCH 166/416] added api limits and fixed tests --- ...CalculatedFieldEntityMessageProcessor.java | 28 ++++++------ ...alculatedFieldManagerMessageProcessor.java | 14 ++++++ ...efaultCalculatedFieldExecutionService.java | 8 ++-- .../cf/ctx/state/CalculatedFieldCtx.java | 3 +- .../cf/ctx/state/RocksDBStateService.java | 2 +- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- .../ctx/state/SingleValueArgumentEntry.java | 20 ++++++--- .../cf/ctx/state/TsRollingArgumentEntry.java | 2 +- .../cf/DefaultTbCalculatedFieldService.java | 18 -------- .../queue/DefaultTbClusterService.java | 6 +-- .../state/ScriptCalculatedFieldStateTest.java | 43 ++++++++++--------- .../state/SimpleCalculatedFieldStateTest.java | 13 +++--- .../state/SingleValueArgumentEntryTest.java | 15 ++++--- .../ctx/state/TsRollingArgumentEntryTest.java | 12 +++--- .../DefaultTenantProfileConfiguration.java | 7 +++ .../server/dao/cf/CalculatedFieldDao.java | 2 + .../CalculatedFieldDataValidator.java | 31 +++++++++++++ .../dao/sql/cf/CalculatedFieldRepository.java | 2 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 +++ 19 files changed, 145 insertions(+), 88 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index f0064ff459..9392e3c02a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -198,20 +198,6 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return state; } - private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { - if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { - return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); - } - return null; - } - - private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { - if (!proto.getTbMsgType().isEmpty()) { - return TbMsgType.valueOf(proto.getTbMsgType()); - } - return null; - } - @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { if (state.isReady()) { @@ -290,4 +276,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return cfIds; } + private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { + if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { + return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); + } + return null; + } + + private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTbMsgType().isEmpty()) { + return TbMsgType.valueOf(proto.getTbMsgType()); + } + return null; + } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index a2577a3f27..d5ed8b8aaa 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -217,6 +217,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + try { + cfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(cf)) { + systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) @@ -257,6 +264,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) if (newCfCtx.hasSignificantChanges(oldCfCtx)) { + try { + newCfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(newCf)) { + systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); + } + } initCf(newCfCtx, callback, true); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index b9a18084b1..d8e42c916d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -61,6 +61,7 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -69,6 +70,7 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; @@ -142,14 +144,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final TimeseriesService timeseriesService; private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; + private final ApiLimitService apiLimitService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap states = new ConcurrentHashMap<>(); - private static final int MAX_LAST_RECORDS_VALUE = 1024; - private static final Set supportedReferencedEntities = EnumSet.of( EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT ); @@ -560,7 +561,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); long startTs = currentTime - timeWindow; - int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + long maxDataPoints = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + int limit = argument.getLimit() == 0 ? (int) maxDataPoints : argument.getLimit(); ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index caeeabea52..a56aed64f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -197,7 +197,8 @@ public class CalculatedFieldCtx { boolean entityIdChanged = !entityId.equals(other.entityId); boolean typeChanged = !cfType.equals(other.cfType); boolean argumentsChanged = !arguments.equals(other.arguments); - return entityIdChanged || typeChanged || argumentsChanged; + boolean expressionChanged = !expression.equals(other.expression); + return entityIdChanged || typeChanged || argumentsChanged || expressionChanged; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index eaa1b08f5e..bfed563182 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -115,7 +115,7 @@ public class RocksDBStateService implements CalculatedFieldStateService { singleValueProtoBuilder.setVersion(entry.getVersion()); } - KvEntry value = entry.getValue(); + KvEntry value = entry.getKvEntryValue(); if (value != null) { singleValueProtoBuilder.setHasV(true) .setValue(ProtoUtils.toKeyValueProto(value)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index d233b60512..01cf9cff70 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -53,7 +53,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { for (Map.Entry entry : this.arguments.entrySet()) { try { - BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getValue(); + BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); } catch (NumberFormatException e) { throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 8d4d40d39b..0832e53e5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -34,19 +35,19 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); private long ts; - private BasicKvEntry value; + private BasicKvEntry kvEntryValue; private Long version; public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.fromProto(entry.getKv()); + this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); } public SingleValueArgumentEntry(AttributeValueProto entry) { this.ts = entry.getLastUpdateTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.basicKvEntryFromProto(entry); + this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); } public SingleValueArgumentEntry(KvEntry entry) { @@ -57,7 +58,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.ts = attributeKvEntry.getLastUpdateTs(); this.version = attributeKvEntry.getVersion(); } - this.value = ProtoUtils.basicKvEntryFromKvEntry(entry); + this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); } /** @@ -65,7 +66,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { * */ private SingleValueArgumentEntry(int ignored) { this.ts = System.currentTimeMillis(); - this.value = null; + this.kvEntryValue = null; } @Override @@ -73,9 +74,14 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return ArgumentEntryType.SINGLE_VALUE; } + @JsonIgnore + public Object getValue() { + return kvEntryValue.getValue(); + } + @Override public ArgumentEntry copy() { - return new SingleValueArgumentEntry(this.ts, this.value, this.version); + return new SingleValueArgumentEntry(this.ts, this.kvEntryValue, this.version); } @Override @@ -88,7 +94,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { Long newVersion = singleValueEntry.getVersion(); if (newVersion == null || this.version == null || newVersion > this.version) { this.ts = singleValueEntry.getTs(); - this.value = singleValueEntry.getValue(); + this.kvEntryValue = singleValueEntry.getKvEntryValue(); this.version = newVersion; return true; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 6c1a772c44..ddae0c3513 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -98,7 +98,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } private boolean updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { - return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getValue()); + return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getKvEntryValue()); } private boolean addTsRecordIfAbsent(Long ts, KvEntry value) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 184092720c..1ccfe2375c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -48,9 +48,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { - private static final int MAX_ARGUMENT_SIZE = 10; - private static final int MAX_CALCULATED_FIELD_NUMBER = 10; - private final CalculatedFieldService calculatedFieldService; @Override @@ -62,9 +59,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); checkForEntityChange(existingCf, calculatedField); } - checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId()); - checkArgumentSize(calculatedField.getConfiguration()); checkReferencedEntities(calculatedField.getConfiguration(), user); CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); @@ -129,19 +124,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } - private void checkArgumentSize(CalculatedFieldConfiguration calculatedFieldConfig) { - if (calculatedFieldConfig.getArguments().size() > MAX_ARGUMENT_SIZE) { - throw new IllegalArgumentException("Too many arguments: " + calculatedFieldConfig.getArguments().size() + ". Max number of argument is " + MAX_ARGUMENT_SIZE); - } - } - - private void checkCalculatedFieldNumber(TenantId tenantId, EntityId entityId) { - int numberOfCalculatedFieldsByEntityId = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, entityId).size(); - if (numberOfCalculatedFieldsByEntityId >= MAX_CALCULATED_FIELD_NUMBER) { - throw new IllegalArgumentException("Max number of calculated fields for entity is " + MAX_CALCULATED_FIELD_NUMBER); - } - } - private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index b272694b16..944e24480e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -43,7 +43,6 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -103,7 +102,6 @@ import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -729,13 +727,13 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback) { - var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } @Override public void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback) { - var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), ComponentLifecycleEvent.DELETED); + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index cbaea6575c..42ff828dbd 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -34,6 +34,8 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -51,7 +53,7 @@ public class ScriptCalculatedFieldStateTest { private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); - private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 43, 122L); + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("assetHumidity", 43L), 122L); private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); private final long ts = System.currentTimeMillis(); @@ -65,6 +67,7 @@ public class ScriptCalculatedFieldStateTest { @BeforeEach void setUp() { ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService); + ctx.init(); state = new ScriptCalculatedFieldState(ctx.getArgNames()); } @@ -93,7 +96,7 @@ public class ScriptCalculatedFieldStateTest { void testUpdateStateWhenUpdateExistingEntry() { state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); - SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, 41, 349L); + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L); Map newArgs = Map.of("assetHumidity", newArgEntry); boolean stateUpdated = state.updateState(newArgs); @@ -116,17 +119,17 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43)); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43L)); } @Test void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException { TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 40000, 4);// will not be used for calculation - values.put(ts - 45000, 2);// will not be used for calculation - values.put(ts - 20, 0); + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, new LongDataEntry("deviceTemperature", 4L));// will not be used for calculation + values.put(ts - 45000, new LongDataEntry("deviceTemperature", 2L));// will not be used for calculation + values.put(ts - 20, new LongDataEntry("deviceTemperature", 0L)); argumentEntry.setTsRecords(values); @@ -138,19 +141,19 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); } @Test void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException { TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 20, 1000);// will not be used - values.put(ts - 18, 0); - values.put(ts - 16, 0); - values.put(ts - 14, 0); - values.put(ts - 12, 0); - values.put(ts - 10, 0); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, new LongDataEntry("deviceTemperature", 1000L));// will not be used + values.put(ts - 18, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 16, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 14, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 12, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 10, new LongDataEntry("deviceTemperature", 0L)); argumentEntry.setTsRecords(values); state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); @@ -161,7 +164,7 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); } @Test @@ -187,10 +190,10 @@ public class ScriptCalculatedFieldStateTest { TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); long ts = System.currentTimeMillis(); - TreeMap values = new TreeMap<>(); - values.put(ts - 40, 10); - values.put(ts - 30, 12); - values.put(ts - 20, 17); + TreeMap values = new TreeMap<>(); + values.put(ts - 40, new LongDataEntry("deviceTemperature", 10L)); + values.put(ts - 30, new LongDataEntry("deviceTemperature", 12L)); + values.put(ts - 20, new LongDataEntry("deviceTemperature", 17L)); argumentEntry.setTsRecords(values); return argumentEntry; diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index 58a981824c..d6b384d85b 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -30,6 +30,8 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -46,9 +48,9 @@ public class SimpleCalculatedFieldStateTest { private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); - private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 11, 145L); - private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, 15, 165L); - private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, 23, 184L); + private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("key1", 11L), 145L); + private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new LongDataEntry("key2", 15L), 165L); + private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new LongDataEntry("key3", 23L), 184L); private SimpleCalculatedFieldState state; private CalculatedFieldCtx ctx; @@ -56,6 +58,7 @@ public class SimpleCalculatedFieldStateTest { @BeforeEach void setUp() { ctx = new CalculatedFieldCtx(getCalculatedField(), null); + ctx.init(); state = new SimpleCalculatedFieldState(ctx.getArgNames()); } @@ -88,7 +91,7 @@ public class SimpleCalculatedFieldStateTest { void testUpdateStateWhenUpdateExistingEntry() { state.arguments = new HashMap<>(Map.of("key1", key1ArgEntry)); - SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), 18, 190L); + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L); Map newArgs = Map.of("key1", newArgEntry); boolean stateUpdated = state.updateState(newArgs); @@ -130,7 +133,7 @@ public class SimpleCalculatedFieldStateTest { void testPerformCalculationWhenPassedNotNumber() { state.arguments = new HashMap<>(Map.of( "key1", key1ArgEntry, - "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, "string", 124L), + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, new StringDataEntry("key2", "string"), 124L), "key3", key3ArgEntry )); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 285da0b423..203d7b3d71 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.LongDataEntry; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -29,7 +30,7 @@ public class SingleValueArgumentEntryTest { @BeforeEach void setUp() { - entry = new SingleValueArgumentEntry(ts, 11, 363L); + entry = new SingleValueArgumentEntry(ts, new LongDataEntry("key", 11L), 363L); } @Test @@ -46,26 +47,26 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWithThaSameTs() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, 13, 363L))).isFalse(); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, new LongDataEntry("key", 13L), 363L))).isFalse(); } @Test void testUpdateEntryWhenNewVersionIsNull() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, 13, null))).isTrue(); - assertThat(entry.getValue()).isEqualTo(13); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, new LongDataEntry("key", 13L), null))).isTrue(); + assertThat(entry.getValue()).isEqualTo(13L); assertThat(entry.getVersion()).isNull(); } @Test void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 369L))).isTrue(); - assertThat(entry.getValue()).isEqualTo(18); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 369L))).isTrue(); + assertThat(entry.getValue()).isEqualTo(18L); assertThat(entry.getVersion()).isEqualTo(369L); } @Test void testUpdateEntryWhenNewVersionIsLessThanCurrent() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 234L))).isFalse(); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 234L))).isFalse(); } } \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index 9ca242092d..5a6f5e96e5 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -54,7 +54,7 @@ public class TsRollingArgumentEntryTest { assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(4); - assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23); + assertThat(entry.getTsRecords().get(ts - 10).getValue()).isEqualTo(23.0); } @Test @@ -76,11 +76,11 @@ public class TsRollingArgumentEntryTest { assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(5); assertThat(entry.getTsRecords()).isEqualTo(Map.of( - ts - 40, 10, - ts - 30, 12, - ts - 20, 17, - ts - 10, 7, - ts - 5, 1 + ts - 40, new DoubleDataEntry("key", 10.0), + ts - 30, new DoubleDataEntry("key", 12.0), + ts - 20, new DoubleDataEntry("key", 17.0), + ts - 10, new DoubleDataEntry("key", 7.0), + ts - 5, new DoubleDataEntry("key", 1.0) )); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 6aa9075a79..0fd630fead 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -135,6 +135,12 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private double warnThreshold; + private long maxCalculatedFieldsPerTenant; + private long maxCalculatedFieldsPerEntity; + private long maxArgumentsPerCF; + private long maxDataPointsPerRollingArg; + private long maxStateSizeInKBytes; + @Override public long getProfileThreshold(ApiUsageRecordKey key) { return switch (key) { @@ -175,6 +181,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura case DASHBOARD -> maxDashboards; case RULE_CHAIN -> maxRuleChains; case EDGE -> maxEdges; + case CALCULATED_FIELD -> maxCalculatedFieldsPerTenant; default -> 0; }; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 23a2eae93e..3efb4011ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -43,4 +43,6 @@ public interface CalculatedFieldDao extends Dao { boolean existsByEntityId(TenantId tenantId, EntityId entityId); + long countCFByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 80e421f350..db8997ceb8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -17,11 +17,15 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; @Component public class CalculatedFieldDataValidator extends DataValidator { @@ -29,13 +33,40 @@ public class CalculatedFieldDataValidator extends DataValidator @Autowired private CalculatedFieldDao calculatedFieldDao; + @Autowired + private ApiLimitService apiLimitService; + + @Override + protected void validateCreate(TenantId tenantId, CalculatedField calculatedField) { + validateNumberOfEntitiesPerTenant(tenantId, EntityType.CALCULATED_FIELD); + validateNumberOfCFsPerEntity(tenantId, calculatedField.getEntityId()); + validateNumberOfArgumentsPerCF(tenantId, calculatedField); + } + @Override protected CalculatedField validateUpdate(TenantId tenantId, CalculatedField calculatedField) { CalculatedField old = calculatedFieldDao.findById(calculatedField.getTenantId(), calculatedField.getId().getId()); if (old == null) { throw new DataValidationException("Can't update non existing calculated field!"); } + validateNumberOfArgumentsPerCF(tenantId, calculatedField); return old; } + private void validateNumberOfCFsPerEntity(TenantId tenantId, EntityId entityId) { + long maxCFsPerEntity = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxCalculatedFieldsPerEntity); + long countCFByEntityId = calculatedFieldDao.countCFByEntityId(tenantId, entityId); + + if (countCFByEntityId == maxCFsPerEntity) { + throw new DataValidationException("Calculated fields per entity limit reached!"); + } + } + + private void validateNumberOfArgumentsPerCF(TenantId tenantId, CalculatedField calculatedField) { + long maxArgumentsPerCF = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxArgumentsPerCF); + if (calculatedField.getConfiguration().getArguments().size() > maxArgumentsPerCF) { + throw new DataValidationException("Calculated field arguments limit reached!"); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index c0118d4f02..2aeca659bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -38,4 +38,6 @@ public interface CalculatedFieldRepository extends JpaRepository removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + long countByTenantIdAndEntityId(UUID tenantId, UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 34bfa27b16..703bdfbf6f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -88,6 +88,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao getEntityClass() { return CalculatedFieldEntity.class; From e62e7fb20be59db8de4cfc7a61fa2c2a6e44ea56 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 3 Feb 2025 12:26:32 +0200 Subject: [PATCH 167/416] implemented linkMatches() method and deletion of state from db --- .../CalculatedFieldManagerMessageProcessor.java | 2 +- .../server/service/cf/RocksDBService.java | 4 ++-- .../service/cf/ctx/state/CalculatedFieldCtx.java | 15 +++++++++++++-- .../service/cf/ctx/state/RocksDBStateService.java | 3 +-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index d5ed8b8aaa..b2ff883236 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -122,7 +122,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (calculatedFields.containsKey(msg.getId().cfId())) { getOrCreateActor(msg.getId().entityId()).tell(msg); } else { - // TODO: remove state from storage + cfExecService.deleteStateFromStorage(msg.getId(), msg.getCallback()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index 5eaeff120a..fe800f61ad 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -59,9 +59,9 @@ public class RocksDBService { } } - public void delete(String key) { + public void delete(CalculatedFieldEntityCtxIdProto key) { try { - db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + db.delete(writeOptions, key.toByteArray()); } catch (RocksDBException e) { log.error("Failed to delete data from RocksDB", e); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index a56aed64f7..4600628838 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -185,8 +186,18 @@ public class CalculatedFieldCtx { } public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { - //TODO: IM - implement - return true; + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return linkMatches(entityId, updatedTelemetry); + } else { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return linkMatches(entityId, updatedTelemetry, scope); + } } public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index bfed563182..6487ce1a43 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -67,7 +66,7 @@ public class RocksDBStateService implements CalculatedFieldStateService { @Override public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + rocksDBService.delete(toProto(ctxId)); callback.onSuccess(); } From 0590b5b271c0f464f8ce7ec75f22c669b2ebb979 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Mon, 3 Feb 2025 17:54:11 +0200 Subject: [PATCH 168/416] Improvements, refactoring, review comments resolve --- .../calculated-fields-table-config.ts | 36 ++-- .../calculated-fields-table.component.ts | 3 + ...lated-field-arguments-table.component.html | 184 +++++++++--------- ...lated-field-arguments-table.component.scss | 13 +- ...culated-field-arguments-table.component.ts | 44 ++--- .../calculated-field-dialog.component.html | 15 +- .../calculated-field-dialog.component.ts | 32 +-- ...ulated-field-argument-panel.component.html | 131 +++++-------- ...ulated-field-argument-panel.component.scss | 33 ---- ...lculated-field-argument-panel.component.ts | 22 +-- .../entity-debug-settings-button.component.ts | 1 - .../entity-debug-settings-panel.component.ts | 6 +- .../entity/entities-table.component.ts | 4 +- .../entity/entity-table-component.models.ts | 1 - .../pages/device/device-tabs.component.html | 3 +- .../entity/entity-autocomplete.component.html | 12 +- .../entity/entity-autocomplete.component.ts | 4 +- .../entity-key-autocomplete.component.ts | 22 ++- .../shared/models/calculated-field.models.ts | 14 +- ui-ngx/src/app/shared/models/constants.ts | 1 + .../assets/locale/locale.constant-en_US.json | 4 + 21 files changed, 265 insertions(+), 320 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 58caa759da..d8c02558f8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -19,7 +19,7 @@ import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.m import { TranslateService } from '@ngx-translate/core'; import { Direction } from '@shared/models/page/sort-order'; import { MatDialog } from '@angular/material/dialog'; -import { TimePageLink } from '@shared/models/page/page-link'; +import { PageLink } from '@shared/models/page/page-link'; import { Observable, of } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; @@ -27,7 +27,7 @@ import { MINUTE } from '@shared/models/time/time.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { getCurrentAuthState, getCurrentAuthUser } from '@core/auth/auth.selectors'; -import { DestroyRef } from '@angular/core'; +import { DestroyRef, Renderer2 } from '@angular/core'; import { EntityDebugSettings } from '@shared/models/entity.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @@ -35,10 +35,10 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap } from 'rxjs/operators'; -import { CalculatedField } from '@shared/models/calculated-field.models'; +import { CalculatedField, CalculatedFieldDialogData } from '@shared/models/calculated-field.models'; import { CalculatedFieldDialogComponent } from './components/public-api'; -export class CalculatedFieldsTableConfig extends EntityTableConfig { +export class CalculatedFieldsTableConfig extends EntityTableConfig { // TODO: [Calculated Fields] remove hardcode when BE variable implemented readonly calculatedFieldsDebugPerTenantLimitsConfiguration = @@ -54,20 +54,16 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); + this.entitiesFetchFunction = (pageLink: PageLink) => this.fetchCalculatedFields(pageLink); this.addEntity = this.addCalculatedField.bind(this); this.deleteEntityTitle = (field: CalculatedField) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); @@ -102,12 +98,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { + fetchCalculatedFields(pageLink: PageLink): Observable> { return this.calculatedFieldsService.getCalculatedFields(this.entityId, pageLink); } onOpenDebugConfig($event: Event, { debugSettings = {}, id }: CalculatedField): void { - const { renderer, viewContainerRef } = this.getTable(); + const { viewContainerRef } = this.getTable(); if ($event) { $event.stopPropagation(); } @@ -115,7 +111,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { this.onDebugConfigChanged(id.id, settings); debugStrategyPopover.hide(); @@ -133,17 +128,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { + return this.getCalculatedFieldDialog() .pipe( filter(Boolean), switchMap(calculatedField => this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField })), ) - .subscribe((res) => { - if (res) { - this.updateData(); - } - }); } private editCalculatedField(calculatedField: CalculatedField): void { @@ -159,8 +149,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { - return this.dialog.open(CalculatedFieldDialogComponent, { + private getCalculatedFieldDialog(value?: CalculatedField, buttonTitle = 'action.add'): Observable { + return this.dialog.open(CalculatedFieldDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 4fda1cc075..bc979a5f0d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -21,6 +21,7 @@ import { DestroyRef, effect, input, + Renderer2, ViewChild, } from '@angular/core'; import { EntityId } from '@shared/models/id/entity-id'; @@ -56,6 +57,7 @@ export class CalculatedFieldsTableComponent { private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, private cd: ChangeDetectorRef, + private renderer: Renderer2, private destroyRef: DestroyRef) { effect(() => { @@ -69,6 +71,7 @@ export class CalculatedFieldsTableComponent { this.durationLeft, this.popoverService, this.destroyRef, + this.renderer ); this.cd.markForCheck(); } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 4f1d1d3980..d1d9998e5a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -15,96 +15,100 @@ limitations under the License. --> -
-
-
{{ 'calculated-fields.argument-name' | translate }}
-
{{ 'calculated-fields.datasource' | translate }}
-
{{ 'common.type' | translate }}
-
{{ 'entity.key' | translate }}
-
-
- @for (group of argumentsFormArray.controls; track group) { -
- - - - @if (group.get('refEntityId')?.get('id')?.value) { - - - - - {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} - - - - - - } @else { - - - - {{ - (group.get('refEntityId')?.get('entityType')?.value === ArgumentEntityType.Tenant - ? 'calculated-fields.argument-current-tenant' - : 'calculated-fields.argument-current') | translate - }} - - - - } - - - @if (group.get('refEntityKey').get('type').value; as type) { - - - {{ ArgumentTypeTranslations.get(type) | translate }} - - +
+
+
+
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'calculated-fields.datasource' | translate }}
+
{{ 'common.type' | translate }}
+
{{ 'entity.key' | translate }}
+
+
+
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + +
+ @if (group.get('refEntityId')?.get('id')?.value) { + + + + + {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} + + + + + + } @else { + + + + {{ + (group.get('refEntityId')?.get('entityType')?.value === ArgumentEntityType.Tenant + ? 'calculated-fields.argument-current-tenant' + : 'calculated-fields.argument-current') | translate + }} + + + + } +
+ + + @if (group.get('refEntityKey').get('type').value; as type) { + + + {{ ArgumentTypeTranslations.get(type) | translate }} + + + } + + + +
+ {{ group.get('refEntityKey').get('key').value }} +
+ + + +
+ + +
+
+ } @empty { + {{ 'calculated-fields.no-arguments' | translate }} } - - - -
- {{ group.get('refEntityKey').get('key').value }} -
- - - -
- -
-
- } @empty { - {{ 'calculated-fields.no-arguments' | translate }} - } -
- @if (errorText && this.argumentsFormArray.dirty) { - - } -
-
- + @if (errorText && this.argumentsFormArray.dirty) { + + } +
+
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 8695ee4068..73f03dc497 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -14,18 +14,7 @@ * limitations under the License. */ :host ::ng-deep { - .inline-entity-autocomplete { - .mat-mdc-form-field-infix { - padding-top: 8px; - padding-bottom: 8px; - min-height: 40px; - width: auto; - .mdc-text-field__input, .mat-mdc-select { - font-weight: 400; - line-height: 20px; - } - } - + .tb-inline-field { a { font-size: 14px; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index c8dae67aec..328a82184b 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -17,7 +17,6 @@ import { ChangeDetectorRef, Component, - DestroyRef, effect, forwardRef, input, @@ -29,6 +28,7 @@ import { AbstractControl, ControlValueAccessor, FormBuilder, + FormGroup, NG_VALIDATORS, NG_VALUE_ACCESSOR, ValidationErrors, @@ -51,6 +51,7 @@ import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { isDefinedAndNotNull } from '@core/utils'; import { charsWithNumRegex } from '@shared/models/regex.constants'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-calculated-field-arguments-table', @@ -78,20 +79,19 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces errorText = ''; argumentsFormArray = this.fb.array([]); - keysPopupClosed = true; readonly entityTypeTranslations = entityTypeTranslations; readonly ArgumentTypeTranslations = ArgumentTypeTranslations; readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; + private popoverComponent: TbPopoverComponent; private propagateChange: (argumentsObj: Record) => void = () => {}; constructor( private fb: FormBuilder, private popoverService: TbPopoverService, private viewContainerRef: ViewContainerRef, - private destroyRef: DestroyRef, private cd: ChangeDetectorRef, private renderer: Renderer2 ) { @@ -123,6 +123,9 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces manageArgument($event: Event, matButton: MatButton, index?: number): void { $event?.stopPropagation(); + if (this.popoverComponent && !this.popoverComponent.tbHidden) { + this.popoverComponent.hide(); + } const trigger = matButton._elementRef.nativeElement; if (this.popoverService.hasPopover(trigger)) { this.popoverService.hidePopover(trigger); @@ -135,27 +138,22 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, }; - this.keysPopupClosed = false; - const argumentsPanelPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, ctx, {}, {}, {}, true); - argumentsPanelPopover.tbComponentRef.instance.popover = argumentsPanelPopover; - argumentsPanelPopover.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { - argumentsPanelPopover.hide(); + this.popoverComponent.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { + this.popoverComponent.hide(); const formGroup = this.getArgumentFormGroup(value); if (isDefinedAndNotNull(index)) { this.argumentsFormArray.setControl(index, formGroup); } else { this.argumentsFormArray.push(formGroup); } - this.argumentsFormArray.markAsDirty(); + formGroup.markAsDirty(); this.cd.markForCheck(); }); - argumentsPanelPopover.tbHideStart.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { - this.keysPopupClosed = true; - }); } } @@ -171,8 +169,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } private getArgumentsObject(): Record { - return this.argumentsFormArray.controls.reduce((acc, control) => { - const rawValue = control.getRawValue(); + return this.argumentsFormArray.getRawValue().reduce((acc, rawValue) => { const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; acc[argumentName] = argument; return acc; @@ -186,24 +183,15 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { - this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], + const value: CalculatedFieldArgumentValue = { ...argumentsObj[key], - ...(argumentsObj[key].refEntityId ? { - refEntityId: this.fb.group({ - entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], - id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], - }), - } : {}), - refEntityKey: this.fb.group({ - type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], - key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], - }), - }) as AbstractControl); + argumentName: key + }; + this.argumentsFormArray.push(this.getArgumentFormGroup(value), {emitEvent: false}); }); } - private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { + private getArgumentFormGroup(value: CalculatedFieldArgumentValue): FormGroup { return this.fb.group({ ...value, argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index e6adc1b4d8..ca27ac6fd1 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -19,7 +19,7 @@

{{ 'entity.type-calculated-field' | translate}}

-
+
{{ 'common.type' | translate }} - + @for (type of fieldTypes; track type) { {{ CalculatedFieldTypeTranslations.get(type) | translate}} } @@ -102,10 +103,10 @@
{{ 'calculated-fields.output' | translate }}
-
+
{{ 'calculated-fields.output-type' | translate }} - + @for (type of outputTypes; track type) { {{ OutputTypeTranslations.get(type) | translate}} } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index c9f1a22157..c8b2073309 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -18,10 +18,9 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { FormGroup, UntypedFormBuilder, Validators } from '@angular/forms'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; -import { helpBaseUrl } from '@shared/models/constants'; import { CalculatedField, CalculatedFieldConfiguration, @@ -35,7 +34,6 @@ import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; -import { isObject } from '@core/utils'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -50,7 +48,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) + map(configuration => Object.keys(configuration.arguments)) ); readonly OutputTypeTranslations = OutputTypeTranslations; @@ -73,7 +71,6 @@ export class CalculatedFieldDialogComponent extends DialogComponent, protected router: Router, @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDialogData, - public dialogRef: MatDialogRef, - public fb: UntypedFormBuilder) { + protected dialogRef: MatDialogRef, + private fb: FormBuilder) { super(store, router, dialogRef); this.applyDialogData(); this.observeTypeChanges(); @@ -104,14 +101,15 @@ export class CalculatedFieldDialogComponent extends DialogComponent
{{ 'calculated-fields.argument-name' | translate }}
-
- - -
- @if (argumentFormGroup.get('argumentName').touched) { - @if (argumentFormGroup.get('argumentName').hasError('required')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { - - warning - - } - } -
- -
+ + + @if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } +
-
{{ 'entity.entity-type' | translate }}
+
{{ 'entity.entity-type' | translate }}
@for (type of argumentEntityTypes; track type) { @@ -67,34 +61,17 @@
- @if (entityType === ArgumentEntityType.Device || entityType === ArgumentEntityType.Asset) { -
-
{{ 'calculated-fields.device-name' | translate }}
- -
- } @else if (entityType === ArgumentEntityType.Customer) { + @if (ArgumentEntityTypeParamsMap.has(entityType)) {
-
{{ 'calculated-fields.customer-name' | translate }}
+
{{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
} @@ -123,13 +100,13 @@ @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
{{ 'calculated-fields.timeseries-key' | translate }}
- +
} @else {
{{ 'calculated-fields.attribute-scope' | translate }}
- - + + {{ 'calculated-fields.server-attributes' | translate }} @@ -149,7 +126,7 @@
{{ 'calculated-fields.attribute-key' | translate }}
{{ 'calculated-fields.default-value' | translate }}
-
- - - -
+ + +
} @else { -
-
{{ 'calculated-fields.time-window' | translate }}
-
- -
+
+
{{ 'calculated-fields.time-window' | translate }}
+
{{ 'calculated-fields.limit' | translate }}
- +
}
-
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 73f03dc497..8f8b1d5d81 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -19,4 +19,9 @@ font-size: 14px; } } + .edit-hint { + .mat-mdc-form-field-error { + font-size: 16px; + } + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 328a82184b..a004e3db6f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -21,7 +21,9 @@ import { forwardRef, input, Input, + OnChanges, Renderer2, + SimpleChanges, ViewContainerRef, } from '@angular/core'; import { @@ -70,10 +72,11 @@ import { TbPopoverComponent } from '@shared/components/popover.component'; } ], }) -export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator { +export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator, OnChanges { @Input() entityId: EntityId; @Input() tenantId: string; + @Input() entityName: string; calculatedFieldType = input() @@ -84,6 +87,8 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces readonly ArgumentTypeTranslations = ArgumentTypeTranslations; readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; + readonly ArgumentType = ArgumentType; + readonly CalculatedFieldType = CalculatedFieldType; private popoverComponent: TbPopoverComponent; private propagateChange: (argumentsObj: Record) => void = () => {}; @@ -105,6 +110,13 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }); } + ngOnChanges(changes: SimpleChanges): void { + if (changes.calculatedFieldType?.previousValue + && changes.calculatedFieldType.currentValue !== changes.calculatedFieldType.previousValue) { + this.argumentsFormArray.markAsDirty(); + } + } + registerOnChange(fn: (argumentsObj: Record) => void): void { this.propagateChange = fn; } @@ -137,6 +149,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces calculatedFieldType: this.calculatedFieldType(), buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, + entityName: this.entityName, }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, @@ -202,6 +215,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }), } : {}), refEntityKey: this.fb.group({ + ...value.refEntityKey, type: [{ value: value.refEntityKey.type, disabled: true }], key: [{ value: value.refEntityKey.key, disabled: true }], }), diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index ca27ac6fd1..37e96cfd91 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -69,6 +69,7 @@ formControlName="arguments" [entityId]="data.entityId" [tenantId]="data.tenantId" + [entityName]="data.entityName" [calculatedFieldType]="fieldFormGroup.get('type').value" />
@@ -114,7 +115,7 @@ @if (outputFormGroup.get('type').value === OutputType.Attribute) { - {{ 'calculated-fields.output-type' | translate }} + {{ 'calculated-fields.attribute-scope' | translate }} {{ 'calculated-fields.server-attributes' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index c8b2073309..155a384273 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -36,6 +36,7 @@ import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { merge } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', @@ -59,10 +60,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(configuration.arguments)) + startWith(null), + map(() => Object.keys(this.configFormGroup.get('arguments').value ?? this.data.value.configuration.arguments)) ); readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 56568a7bfe..d274c30b8f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -96,7 +96,7 @@ }
- @if (entityFilter.singleEntity.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { + @if (entityFilter.singleEntity?.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
{{ 'calculated-fields.timeseries-key' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 60d79d7bd6..b47d2f073c 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -49,6 +49,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { @Input() argument: CalculatedFieldArgumentValue; @Input() entityId: EntityId; @Input() tenantId: string; + @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); @@ -83,6 +84,8 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { readonly ArgumentEntityType = ArgumentEntityType; readonly ArgumentEntityTypeParamsMap = ArgumentEntityTypeParamsMap; + private currentEntityFilter: EntityFilter; + constructor( private fb: FormBuilder, private cd: ChangeDetectorRef, @@ -107,6 +110,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); + this.currentEntityFilter = this.getCurrentEntityFilter(); this.updateEntityFilter(this.argument.refEntityId?.entityType, true); this.toggleByEntityKeyType(this.argument.refEntityKey?.type); this.setInitialEntityKeyType(); @@ -138,30 +142,53 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { } private updateEntityFilter(entityType: ArgumentEntityType = ArgumentEntityType.Current, onInit = false): void { - let entityId: EntityId; + let entityFilter: EntityFilter; switch (entityType) { case ArgumentEntityType.Current: - entityId = this.entityId + entityFilter = this.currentEntityFilter; break; case ArgumentEntityType.Tenant: - entityId = { - id: this.tenantId, - entityType: EntityType.TENANT + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: { + id: this.tenantId, + entityType: EntityType.TENANT + }, }; break; default: - entityId = this.argumentFormGroup.get('refEntityId').value as unknown as EntityId; + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: this.argumentFormGroup.get('refEntityId').value as unknown as EntityId, + }; } if (!onInit) { this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); } - this.entityFilter = { - type: AliasFilterType.singleEntity, - singleEntity: entityId, - }; + this.entityFilter = entityFilter; this.cd.markForCheck(); } + private getCurrentEntityFilter(): EntityFilter { + switch (this.entityId.entityType) { + case EntityType.ASSET_PROFILE: + return { + deviceTypes: [this.entityName], + type: AliasFilterType.assetType + }; + case EntityType.DEVICE_PROFILE: + return { + deviceTypes: [this.entityName], + type: AliasFilterType.deviceType + }; + default: + return { + type: AliasFilterType.singleEntity, + singleEntity: this.entityId, + }; + } + } + private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, diff --git a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html index 9084301783..f70d4f443b 100644 --- a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html @@ -15,6 +15,10 @@ limitations under the License. --> + + + diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index 4329cd7e3c..2b677dc278 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -32,6 +32,10 @@ [entityName]="entity.name"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index aa928687ad..37cfafa9aa 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -69,6 +69,10 @@
+ + + diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 533384e1c4..f73926f62b 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -120,6 +120,7 @@ export interface CalculatedFieldDialogData { entityId: EntityId; debugLimitsConfiguration: string; tenantId: string; + entityName?: string; } export interface ArgumentEntityTypeParams { From d14d0d4e8a8f011df869fedd96c9a09964c60af4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Feb 2025 17:03:25 +0200 Subject: [PATCH 177/416] used limits for state persistence --- .../server/actors/ActorSystemContext.java | 6 ++++++ .../CalculatedFieldEntityMessageProcessor.java | 4 ++-- .../CalculatedFieldManagerMessageProcessor.java | 6 +++--- .../service/cf/CalculatedFieldExecutionService.java | 2 +- .../service/cf/DefaultCalculatedFieldCache.java | 4 +++- .../cf/DefaultCalculatedFieldExecutionService.java | 4 ++-- .../service/cf/ctx/CalculatedFieldStateService.java | 3 ++- .../cf/ctx/state/BaseCalculatedFieldState.java | 5 +++-- .../service/cf/ctx/state/CalculatedFieldCtx.java | 10 +++++++++- .../service/cf/ctx/state/RocksDBStateService.java | 7 +++++-- .../ctx/state/ScriptCalculatedFieldStateTest.java | 12 ++++++++++-- .../ctx/state/SimpleCalculatedFieldStateTest.java | 13 ++++++++++++- .../common/data/cf/configuration/Argument.java | 6 ++++-- .../server/common/data/cf/configuration/Output.java | 2 ++ .../data/cf/configuration/ReferencedEntityKey.java | 4 ++-- 15 files changed, 66 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index daa6744fd9..49e2c9c46e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -98,6 +98,7 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -516,6 +517,11 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Lazy + @Autowired(required = false) + @Getter + private ApiLimitService apiLimitService; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 9392e3c02a..f6bc5b2d63 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -200,7 +200,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { - if (state.isReady()) { + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { @@ -209,7 +209,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { callback.onSuccess(); // State was updated but no calculation performed; } - cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + cfService.pushStateToStorage(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index ec86e640d4..31c8eee23c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -95,7 +95,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onFieldInitMsg(CalculatedFieldInitMsg msg) { var cf = msg.getCf(); - var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { @@ -220,7 +220,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { - var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { @@ -248,7 +248,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { - var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService()); + var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getId()); List newCfList = new ArrayList<>(oldCfList.size()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 19f60165cf..393fbd3ec2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -43,7 +43,7 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); - void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); + void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 7e841a0cf8..c8f9a7e882 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.Collections; @@ -57,6 +58,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final AssetService assetService; private final DeviceService deviceService; private final TbelInvokeService tbelInvokeService; + private final ApiLimitService apiLimitService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -116,7 +118,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { if (ctx == null) { CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { - ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); + ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); calculatedFieldsCtx.put(calculatedFieldId, ctx); log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d8e42c916d..03c8f3de0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -265,8 +265,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - stateService.persistState(stateId, state, callback); + public void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + stateService.persistState(ctx, stateId, state, callback); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index c670e97580..e822d52767 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf.ctx; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.Map; @@ -24,7 +25,7 @@ public interface CalculatedFieldStateService { Map restoreStates(); - void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); + void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 21105a44cb..86d83a1f70 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; import java.util.ArrayList; @@ -23,6 +24,7 @@ import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected List requiredArguments; @@ -34,8 +36,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } public BaseCalculatedFieldState() { - this.requiredArguments = new ArrayList<>(); - this.arguments = new HashMap<>(); + this(new ArrayList<>(), new HashMap<>()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 4600628838..c483c6ab5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -33,8 +33,10 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -67,7 +69,10 @@ public class CalculatedFieldCtx { private boolean initialized; - public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + private long maxDataPointsPerRollingArg; + private long maxStateSizeInKBytes; + + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { this.calculatedField = calculatedField; this.cfId = calculatedField.getId(); @@ -96,6 +101,9 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; + + this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + this.maxStateSizeInKBytes = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes); } public void init() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 18e604118d..371751cc6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -59,8 +59,11 @@ public class RocksDBStateService implements CalculatedFieldStateService { } @Override - public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - rocksDBService.put(toProto(stateId), toProto(stateId, state)); + public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + CalculatedFieldStateProto stateProto = toProto(stateId, state); + if (stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { + rocksDBService.put(toProto(stateId), toProto(stateId, state)); + } callback.onSuccess(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 42ff828dbd..bded0d058d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; @@ -36,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -45,6 +47,8 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; @SpringBootTest(classes = DefaultTbelInvokeService.class) public class ScriptCalculatedFieldStateTest { @@ -64,9 +68,13 @@ public class ScriptCalculatedFieldStateTest { @Autowired private TbelInvokeService tbelInvokeService; + @MockBean + private ApiLimitService apiLimitService; + @BeforeEach void setUp() { - ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService); + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService, apiLimitService); ctx.init(); state = new ScriptCalculatedFieldState(ctx.getArgNames()); } @@ -219,7 +227,7 @@ public class ScriptCalculatedFieldStateTest { ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temperature", ArgumentType.TS_ROLLING, null); argument1.setRefEntityKey(refEntityKey1); argument1.setLimit(5); - argument1.setTimeWindow(30000); + argument1.setTimeWindow(30000L); Argument argument2 = new Argument(); ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index d6b384d85b..d803ad2dab 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -17,6 +17,9 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -32,6 +35,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -41,7 +45,10 @@ import java.util.concurrent.ExecutionException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +@ExtendWith(MockitoExtension.class) public class SimpleCalculatedFieldStateTest { private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); @@ -55,9 +62,13 @@ public class SimpleCalculatedFieldStateTest { private SimpleCalculatedFieldState state; private CalculatedFieldCtx ctx; + @Mock + private ApiLimitService apiLimitService; + @BeforeEach void setUp() { - ctx = new CalculatedFieldCtx(getCalculatedField(), null); + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), null, apiLimitService); ctx.init(); state = new SimpleCalculatedFieldState(ctx.getArgNames()); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index b61c3bc507..9923751db6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import org.springframework.lang.Nullable; import org.thingsboard.server.common.data.id.EntityId; @Data +@JsonInclude(JsonInclude.Include.NON_NULL) public class Argument { @Nullable @@ -27,7 +29,7 @@ public class Argument { private ReferencedEntityKey refEntityKey; private String defaultValue; - private int limit; - private long timeWindow; + private Integer limit; + private Long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 12cf97338a..b57b19d3ef 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; @Data +@JsonInclude(JsonInclude.Include.NON_NULL) public class Output { private String name; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java index b4bcc77a17..fd0bf3ceb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -15,18 +15,18 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; @Data @AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) public class ReferencedEntityKey { private String key; private ArgumentType type; private AttributeScope scope; - - } From 555acff6b4ced9405b2ef2cfcc37e90a6b4d8fa4 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 17:12:03 +0200 Subject: [PATCH 178/416] fix --- .../panel/calculated-field-argument-panel.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index b47d2f073c..fe60162005 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -173,7 +173,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { switch (this.entityId.entityType) { case EntityType.ASSET_PROFILE: return { - deviceTypes: [this.entityName], + assetTypes: [this.entityName], type: AliasFilterType.assetType }; case EntityType.DEVICE_PROFILE: From 4657e66d92e650f9b1dbaf58e66f554f9bd7b090 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 4 Feb 2025 17:15:43 +0200 Subject: [PATCH 179/416] Fixed assignemnt/unassignemnt of dashboards --- .../dashboard/BaseDashboardProcessor.java | 62 +++++++++++-------- .../dashboard/DashboardEdgeProcessor.java | 6 +- .../dashboard/DashboardEdgeProcessorV1.java | 3 +- .../server/edge/DashboardEdgeTest.java | 15 ++++- 4 files changed, 54 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java index 52277b8527..faf001fbba 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java @@ -26,6 +26,7 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; +import java.util.HashSet; import java.util.Set; @Slf4j @@ -40,54 +41,63 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { if (dashboard == null) { throw new RuntimeException("[{" + tenantId + "}] dashboardUpdateMsg {" + dashboardUpdateMsg + "} cannot be converted to dashboard"); } - Set assignedCustomers = null; + Set newAssignedCustomers = new HashSet<>(dashboard.getAssignedCustomers()); Dashboard dashboardById = edgeCtx.getDashboardService().findDashboardById(tenantId, dashboardId); if (dashboardById == null) { created = true; dashboard.setId(null); + dashboard.setAssignedCustomers(null); } else { dashboard.setId(dashboardId); - assignedCustomers = filterNonExistingCustomers(tenantId, dashboardById.getAssignedCustomers()); + dashboard.setAssignedCustomers(dashboardById.getAssignedCustomers()); } dashboardValidator.validate(dashboard, Dashboard::getTenantId); if (created) { dashboard.setId(dashboardId); } - Set msgAssignedCustomers = filterNonExistingCustomers(tenantId, dashboard.getAssignedCustomers()); - if (msgAssignedCustomers != null) { - if (assignedCustomers == null) { - assignedCustomers = msgAssignedCustomers; - } else { - assignedCustomers.addAll(msgAssignedCustomers); + + Dashboard savedDashboard = edgeCtx.getDashboardService().saveDashboard(dashboard, false); + + updateDashboardAssignments(tenantId, dashboardById, savedDashboard, newAssignedCustomers); + + return created; + } + + private void updateDashboardAssignments(TenantId tenantId, Dashboard dashboardById, Dashboard savedDashboard, Set newAssignedCustomers) { + Set currentAssignedCustomers = new HashSet<>(); + if (dashboardById != null) { + if (dashboardById.getAssignedCustomers() != null) { + currentAssignedCustomers.addAll(dashboardById.getAssignedCustomers()); } } - dashboard.setAssignedCustomers(assignedCustomers); - Dashboard savedDashboard = edgeCtx.getDashboardService().saveDashboard(dashboard, false); - if (msgAssignedCustomers != null && !msgAssignedCustomers.isEmpty()) { - for (ShortCustomerInfo assignedCustomer : msgAssignedCustomers) { - if (assignedCustomer.getCustomerId().equals(customerId)) { - edgeCtx.getDashboardService().assignDashboardToCustomer(tenantId, savedDashboard.getId(), assignedCustomer.getCustomerId()); - } + + newAssignedCustomers = filterNonExistingCustomers(tenantId, currentAssignedCustomers, newAssignedCustomers); + + Set addedCustomerIds = new HashSet<>(); + Set removedCustomerIds = new HashSet<>(); + for (ShortCustomerInfo newAssignedCustomer : newAssignedCustomers) { + if (!savedDashboard.isAssignedToCustomer(newAssignedCustomer.getCustomerId())) { + addedCustomerIds.add(newAssignedCustomer.getCustomerId()); } - } else { - unassignCustomersFromDashboard(tenantId, savedDashboard, customerId); } - return created; - } - private void unassignCustomersFromDashboard(TenantId tenantId, Dashboard dashboard, CustomerId customerId) { - if (dashboard.getAssignedCustomers() != null && !dashboard.getAssignedCustomers().isEmpty()) { - for (ShortCustomerInfo assignedCustomer : dashboard.getAssignedCustomers()) { - if (assignedCustomer.getCustomerId().equals(customerId)) { - edgeCtx.getDashboardService().unassignDashboardFromCustomer(tenantId, dashboard.getId(), assignedCustomer.getCustomerId()); - } + for (ShortCustomerInfo currentAssignedCustomer : currentAssignedCustomers) { + if (!newAssignedCustomers.contains(currentAssignedCustomer)) { + removedCustomerIds.add(currentAssignedCustomer.getCustomerId()); } } + + for (CustomerId customerIdToAdd : addedCustomerIds) { + edgeCtx.getDashboardService().assignDashboardToCustomer(tenantId, savedDashboard.getId(), customerIdToAdd); + } + for (CustomerId customerIdToRemove : removedCustomerIds) { + edgeCtx.getDashboardService().unassignDashboardFromCustomer(tenantId, savedDashboard.getId(), customerIdToRemove); + } } protected abstract Dashboard constructDashboardFromUpdateMsg(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg); - protected abstract Set filterNonExistingCustomers(TenantId tenantId, Set assignedCustomers); + protected abstract Set filterNonExistingCustomers(TenantId tenantId, Set currentAssignedCustomers, Set newAssignedCustomers); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java index cb54977c3c..8d953907b6 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java @@ -128,9 +128,9 @@ public abstract class DashboardEdgeProcessor extends BaseDashboardProcessor impl } @Override - protected Set filterNonExistingCustomers(TenantId tenantId, Set assignedCustomers) { - // do nothing on cloud - return assignedCustomers; + protected Set filterNonExistingCustomers(TenantId tenantId, Set currentAssignedCustomers, Set newAssignedCustomers) { + newAssignedCustomers.addAll(currentAssignedCustomers); + return newAssignedCustomers; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV1.java index 20ab6019bf..c13c85779a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV1.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV1.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; import org.thingsboard.server.queue.util.TbCoreComponent; +import java.util.HashSet; import java.util.Set; @Component @@ -44,7 +45,7 @@ public class DashboardEdgeProcessorV1 extends DashboardEdgeProcessor { Set assignedCustomers; if (dashboardUpdateMsg.hasAssignedCustomers()) { assignedCustomers = JacksonUtil.fromString(dashboardUpdateMsg.getAssignedCustomers(), new TypeReference<>() {}); - assignedCustomers = filterNonExistingCustomers(tenantId, assignedCustomers); + assignedCustomers = filterNonExistingCustomers(tenantId, new HashSet<>(), assignedCustomers); dashboard.setAssignedCustomers(assignedCustomers); } dashboard.setMobileOrder(dashboardUpdateMsg.hasMobileOrder() ? dashboardUpdateMsg.getMobileOrder() : null); diff --git a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java index 3b252046ab..5b09e05a09 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edge; import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Sets; import com.google.protobuf.AbstractMessage; import org.junit.Assert; import org.junit.Test; @@ -175,7 +176,11 @@ public class DashboardEdgeTest extends AbstractEdgeTest { @Test public void testSendDashboardToCloud() throws Exception { - Dashboard dashboard = buildDashboardForUplinkMsg(); + Customer customer = new Customer(); + customer.setTitle("Edge Customer"); + Customer savedCustomer = doPost("/api/customer", customer, Customer.class); + + Dashboard dashboard = buildDashboardForUplinkMsg(savedCustomer); UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); DashboardUpdateMsg.Builder dashboardUpdateMsgBuilder = DashboardUpdateMsg.newBuilder(); @@ -196,6 +201,11 @@ public class DashboardEdgeTest extends AbstractEdgeTest { Dashboard foundDashboard = doGet("/api/dashboard/" + dashboard.getUuidId(), Dashboard.class); Assert.assertNotNull(foundDashboard); Assert.assertEquals("Edge Test Dashboard", foundDashboard.getName()); + + PageData pageData = doGetTypedWithPageLink("/api/customer/" + savedCustomer.getId().toString() + "/dashboards?", + new TypeReference<>() {}, new PageLink(100)); + Assert.assertEquals(1, pageData.getData().size()); + Assert.assertEquals("Edge Test Dashboard", pageData.getData().get(0).getTitle()); } @Test @@ -242,11 +252,12 @@ public class DashboardEdgeTest extends AbstractEdgeTest { return savedDashboard; } - private Dashboard buildDashboardForUplinkMsg() { + private Dashboard buildDashboardForUplinkMsg(Customer savedCustomer) { Dashboard dashboard = new Dashboard(); dashboard.setId(new DashboardId(UUID.randomUUID())); dashboard.setTenantId(tenantId); dashboard.setTitle("Edge Test Dashboard"); + dashboard.setAssignedCustomers(Sets.newHashSet(new ShortCustomerInfo(savedCustomer.getId(), savedCustomer.getTitle(), savedCustomer.isPublic()))); return dashboard; } From be41d477f8bc5b8b619006ad6773b0faf847b0ff Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 4 Feb 2025 17:33:57 +0200 Subject: [PATCH 180/416] UI: Fixed detect change state in dashboard --- .../widget/lib/action/action-widget.models.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts index dbf83aac9a..9c5bc7885a 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/action/action-widget.models.ts @@ -25,6 +25,7 @@ import { WidgetContext } from '@home/models/widget-component.models'; import { BehaviorSubject, forkJoin, + merge, Observable, Observer, of, @@ -33,7 +34,7 @@ import { switchMap, throwError } from 'rxjs'; -import { catchError, delay, map, share, take } from 'rxjs/operators'; +import { catchError, debounceTime, delay, map, share, take } from 'rxjs/operators'; import { AfterViewInit, ChangeDetectorRef, Directive, Input, OnDestroy, OnInit, TemplateRef } from '@angular/core'; import { DataToValueSettings, @@ -657,8 +658,15 @@ export class DashboardStateWithParamsGetter extends ValueGetter { if (this.simulated) { return of({id: 'default', params: {}}); } else { - return this.ctx.stateController.dashboardCtrl.dashboardCtx.stateId.pipe( - map(id => ({id, params: deepClone(this.ctx.stateController.getStateParams())})) + return merge( + this.ctx.stateController.dashboardCtrl.dashboardCtx.stateId, + this.ctx.stateController.dashboardCtrl.dashboardCtx.stateChanged + ).pipe( + debounceTime(10), + map(() => ({ + id: this.ctx.stateController.getStateId(), + params: deepClone(this.ctx.stateController.getStateParams()) + })) ); } } From d185b784274f9ed19302c6bce1e5976fb29bd3ff Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 17:57:05 +0200 Subject: [PATCH 181/416] Resolved PR comments --- ...lated-field-arguments-table.component.html | 19 ++++++++------ ...lated-field-arguments-table.component.scss | 9 ++++--- .../calculated-field-dialog.component.ts | 7 +++--- ...lculated-field-argument-panel.component.ts | 25 +++---------------- .../shared/models/calculated-field.models.ts | 21 ++++++++++++++++ 5 files changed, 44 insertions(+), 37 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 24d67d8654..b25809ac82 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -87,19 +87,24 @@ (click)="manageArgument($event, button, $index)" [matTooltip]="'action.edit' | translate" matTooltipPosition="above"> - edit - @if (argumentsFormArray.dirty - && group.get('refEntityKey').get('type').value === ArgumentType.Rolling - && calculatedFieldType() === CalculatedFieldType.SIMPLE) { - - } + + edit +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 8f8b1d5d81..3a1f1fbe25 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -19,9 +19,10 @@ font-size: 14px; } } - .edit-hint { - .mat-mdc-form-field-error { - font-size: 16px; - } +} + +:host { + .field-action { + color: rgba(0, 0, 0, 0.54); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 155a384273..55fc299475 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -36,7 +36,6 @@ import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; -import { merge } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', @@ -60,10 +59,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(this.configFormGroup.get('arguments').value ?? this.data.value.configuration.arguments)) + startWith(this.data.value?.configuration?.arguments ?? {}), + map(argumentsObj => Object.keys(argumentsObj)) ); readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index fe60162005..510bdd95f3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -25,7 +25,8 @@ import { ArgumentType, ArgumentTypeTranslations, CalculatedFieldArgumentValue, - CalculatedFieldType + CalculatedFieldType, + getCalculatedFieldCurrentEntityFilter } from '@shared/models/calculated-field.models'; import { debounceTime, distinctUntilChanged, filter } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; @@ -110,7 +111,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); - this.currentEntityFilter = this.getCurrentEntityFilter(); + this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); this.updateEntityFilter(this.argument.refEntityId?.entityType, true); this.toggleByEntityKeyType(this.argument.refEntityKey?.type); this.setInitialEntityKeyType(); @@ -169,26 +170,6 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { this.cd.markForCheck(); } - private getCurrentEntityFilter(): EntityFilter { - switch (this.entityId.entityType) { - case EntityType.ASSET_PROFILE: - return { - assetTypes: [this.entityName], - type: AliasFilterType.assetType - }; - case EntityType.DEVICE_PROFILE: - return { - deviceTypes: [this.entityName], - type: AliasFilterType.deviceType - }; - default: - return { - type: AliasFilterType.singleEntity, - singleEntity: this.entityId, - }; - } - } - private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index f73926f62b..fac1e9f942 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -20,6 +20,7 @@ import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; +import { AliasFilterType } from '@shared/models/alias.models'; export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { debugSettings?: EntityDebugSettings; @@ -133,3 +134,23 @@ export const ArgumentEntityTypeParamsMap =new Map { + switch (entityId.entityType) { + case EntityType.ASSET_PROFILE: + return { + assetTypes: [entityName], + type: AliasFilterType.assetType + }; + case EntityType.DEVICE_PROFILE: + return { + deviceTypes: [entityName], + type: AliasFilterType.deviceType + }; + default: + return { + type: AliasFilterType.singleEntity, + singleEntity: entityId, + }; + } +} From 33825e5660e5aeb39271904691a503d5c2232fd2 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 18:24:03 +0200 Subject: [PATCH 182/416] Changed styling --- ...calculated-field-arguments-table.component.html | 7 +++---- ...calculated-field-arguments-table.component.scss | 14 ++++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index b25809ac82..4b7e516db0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -80,7 +80,7 @@ -
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 3a1f1fbe25..188371b0d8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -13,6 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +:host { + .tb-form-table-row-cell-buttons{ + --mat-badge-legacy-small-size-container-size: 8px; + --mat-badge-small-size-container-overlap-offset: -5px; + --mat-badge-small-size-text-size: 0; + } +} + :host ::ng-deep { .tb-inline-field { a { @@ -20,9 +28,3 @@ } } } - -:host { - .field-action { - color: rgba(0, 0, 0, 0.54); - } -} From 1ac9d4137773e539880b6ba51063d8c5ddeac499 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 18:24:34 +0200 Subject: [PATCH 183/416] formatting --- .../calculated-field-arguments-table.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 188371b0d8..a0c90bba32 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - .tb-form-table-row-cell-buttons{ + .tb-form-table-row-cell-buttons { --mat-badge-legacy-small-size-container-size: 8px; --mat-badge-small-size-container-overlap-offset: -5px; --mat-badge-small-size-text-size: 0; From 76155aca75c3e49553700b354941779ff1472f5f Mon Sep 17 00:00:00 2001 From: Oleksandra Matviienko Date: Tue, 4 Feb 2025 21:24:24 +0100 Subject: [PATCH 184/416] Added test to verify attribute and RPC subscriptions removal on session overflow. Signed-off-by: Oleksandra Matviienko --- .../server/actors/ActorSystemContext.java | 2 +- .../device/DeviceActorMessageProcessor.java | 4 +-- .../DeviceActorMessageProcessorTest.java | 30 ++++++++++++++++++- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 1c18a31f68..b13cf8e490 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -489,7 +489,7 @@ public class ActorSystemContext { @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter - private long maxConcurrentSessionsPerDevice; + private int maxConcurrentSessionsPerDevice; @Value("${actors.session.sync.timeout:10000}") @Getter diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 77bf7c32fa..a29fcfc362 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -126,8 +126,8 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso final TenantId tenantId; final DeviceId deviceId; final LinkedHashMapRemoveEldest sessions; - private final Map attributeSubscriptions; - private final Map rpcSubscriptions; + final Map attributeSubscriptions; + final Map rpcSubscriptions; private final Map toDeviceRpcPendingMap; private final boolean rpcSequential; private final RpcSubmitStrategy rpcSubmitStrategy; diff --git a/application/src/test/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessorTest.java b/application/src/test/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessorTest.java index 35f53587e5..8408f9868f 100644 --- a/application/src/test/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessorTest.java +++ b/application/src/test/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessorTest.java @@ -17,22 +17,27 @@ package org.thingsboard.server.actors.device; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.thingsboard.common.util.LinkedHashMapRemoveEldest; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.service.transport.TbCoreToTransportService; + +import java.util.UUID; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; public class DeviceActorMessageProcessorTest { - public static final long MAX_CONCURRENT_SESSIONS_PER_DEVICE = 10L; + public static final int MAX_CONCURRENT_SESSIONS_PER_DEVICE = 10; ActorSystemContext systemContext; DeviceService deviceService; TenantId tenantId = TenantId.SYS_TENANT_ID; @@ -47,6 +52,7 @@ public class DeviceActorMessageProcessorTest { willReturn(MAX_CONCURRENT_SESSIONS_PER_DEVICE).given(systemContext).getMaxConcurrentSessionsPerDevice(); willReturn(deviceService).given(systemContext).getDeviceService(); processor = new DeviceActorMessageProcessor(systemContext, tenantId, deviceId); + willReturn(mock(TbCoreToTransportService.class)).given(systemContext).getTbCoreToTransportService(); } @Test @@ -55,4 +61,26 @@ public class DeviceActorMessageProcessorTest { assertThat(processor.sessions.getMaxEntries(), is(MAX_CONCURRENT_SESSIONS_PER_DEVICE)); assertThat(processor.sessions.getRemovalConsumer(), notNullValue()); } + + @Test + public void givenFullSessionMap_whenSessionOverflow_thenShouldDeleteAttributeAndRPCSubscriptions() { + //givenFullSessionMap + for (int i = 0; i < MAX_CONCURRENT_SESSIONS_PER_DEVICE; i++) { + UUID sessionID = UUID.randomUUID(); + processor.sessions.put(sessionID, Mockito.mock(SessionInfoMetaData.class, RETURNS_DEEP_STUBS)); + processor.attributeSubscriptions.put(sessionID, Mockito.mock(SessionInfo.class)); + processor.rpcSubscriptions.put(sessionID, Mockito.mock(SessionInfo.class)); + } + assertThat(processor.sessions.size(), is(MAX_CONCURRENT_SESSIONS_PER_DEVICE)); + assertThat(processor.attributeSubscriptions.size(), is(MAX_CONCURRENT_SESSIONS_PER_DEVICE)); + assertThat(processor.rpcSubscriptions.size(), is(MAX_CONCURRENT_SESSIONS_PER_DEVICE)); + + //add one more + processor.sessions.put(UUID.randomUUID(), Mockito.mock(SessionInfoMetaData.class)); + + assertThat(processor.sessions.size(), is(MAX_CONCURRENT_SESSIONS_PER_DEVICE)); + assertThat(processor.attributeSubscriptions.size(), is(MAX_CONCURRENT_SESSIONS_PER_DEVICE-1)); + assertThat(processor.rpcSubscriptions.size(), is(MAX_CONCURRENT_SESSIONS_PER_DEVICE-1)); + + } } \ No newline at end of file From 783579a093c7c79d4ecd5d9748c3981878838941 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 5 Feb 2025 11:56:51 +0200 Subject: [PATCH 185/416] UI: Fixed unclear cached result in dashboard autocomplete when changing user --- .../dashboard-autocomplete.component.html | 4 +++- .../components/dashboard-autocomplete.component.ts | 14 ++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ui-ngx/src/app/shared/components/dashboard-autocomplete.component.html b/ui-ngx/src/app/shared/components/dashboard-autocomplete.component.html index bfdf4dce2b..c8c6935619 100644 --- a/ui-ngx/src/app/shared/components/dashboard-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/dashboard-autocomplete.component.html @@ -31,7 +31,9 @@ [required]="required" [matAutocomplete]="dashboardAutocomplete" [class.!hidden]="useDashboardLink && disabled && selectDashboardFormGroup.get('dashboard').value"> - + {{ displayDashboardFn(selectDashboardFormGroup.get('dashboard').value) | customTranslate }} + +
+ +
+
+ +
+ diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts new file mode 100644 index 0000000000..8efb21dbf2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -0,0 +1,53 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { AfterViewInit, Component, Inject, ViewChild } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { DebugEventType, EventType } from '@shared/models/event.models'; +import { EventTableComponent } from '@home/components/event/event-table.component'; +import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-calculated-field-debug-dialog', + templateUrl: './calculated-field-debug-dialog.component.html', +}) +export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { + + @ViewChild(EventTableComponent, {static: true}) eventsTable: EventTableComponent; + + readonly DebugEventType = DebugEventType; + readonly debugEventTypes = DebugEventType; + readonly EventType = EventType; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDebugDialogData, + protected dialogRef: MatDialogRef) { + super(store, router, dialogRef); + } + + ngAfterViewInit(): void { + this.eventsTable.entitiesTable.updateData(); + } + + cancel(): void { + this.dialogRef.close(null); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 37e96cfd91..ac60ae8178 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -51,6 +51,7 @@ [class.mb-5]="fieldFormGroup.get('name').errors && fieldFormGroup.get('name').touched" [entityLabel]="'debug-settings.calculated-field' | translate" [debugLimitsConfiguration]="data.debugLimitsConfiguration" + [additionalActionConfig]="additionalDebugActionConfig" /> diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 55fc299475..3575223764 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -65,6 +65,14 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(argumentsObj)) ); + additionalDebugActionConfig = this.data.value?.id ? { + ...this.data.additionalDebugActionConfig, + action: () => this.data.additionalDebugActionConfig.action({ + ...this.data.additionalDebugActionConfig.data, + id: this.data.value.id, + }), + } : null; + readonly OutputTypeTranslations = OutputTypeTranslations; readonly OutputType = OutputType; readonly AttributeScope = AttributeScope; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts index bc89e4dc6f..78b8862c2e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -17,3 +17,4 @@ export * from './dialog/calculated-field-dialog.component'; export * from './arguments-table/calculated-field-arguments-table.component'; export * from './panel/calculated-field-argument-panel.component'; +export * from './debug-dialog/calculated-field-debug-dialog.component'; diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts index 597fd79b8e..7e06568489 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts @@ -32,7 +32,7 @@ import { EntityDebugSettingsPanelComponent } from './entity-debug-settings-panel import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { BehaviorSubject, of, shareReplay, timer } from 'rxjs'; import { SECOND, MINUTE } from '@shared/models/time/time.models'; -import { EntityDebugSettings } from '@shared/models/entity.models'; +import { AdditionalDebugActionConfig, EntityDebugSettings } from '@shared/models/entity.models'; import { map, switchMap, takeWhile } from 'rxjs/operators'; import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { AppState } from '@core/core.state'; @@ -61,6 +61,7 @@ export class EntityDebugSettingsButtonComponent implements ControlValueAccessor @Input() debugLimitsConfiguration: string; @Input() entityLabel: string; + @Input() additionalActionConfig: AdditionalDebugActionConfig; debugSettingsFormGroup = this.fb.group({ failuresEnabled: [false], @@ -133,7 +134,8 @@ export class EntityDebugSettingsButtonComponent implements ControlValueAccessor ...debugSettings, maxDebugModeDuration: this.maxDebugModeDuration, debugLimitsConfiguration: this.debugLimitsConfiguration, - entityLabel: this.entityLabel + entityLabel: this.entityLabel, + additionalActionConfig: this.additionalActionConfig, }, {}, {}, {}, true); diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html index 7576d4b2fe..dcc3355890 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html @@ -48,20 +48,32 @@ -
- - +
+
+ @if (additionalActionConfig) { + + } +
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts index 72748e27e2..9e8ac1ded6 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts @@ -21,7 +21,7 @@ import { Component, EventEmitter, Input, - OnInit + OnInit, } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { TbPopoverComponent } from '@shared/components/popover.component'; @@ -32,7 +32,7 @@ import { SECOND } from '@shared/models/time/time.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { of, shareReplay, timer } from 'rxjs'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { EntityDebugSettings } from '@shared/models/entity.models'; +import { AdditionalDebugActionConfig, EntityDebugSettings } from '@shared/models/entity.models'; import { distinctUntilChanged, map, startWith, switchMap, takeWhile } from 'rxjs/operators'; @Component({ @@ -54,6 +54,7 @@ export class EntityDebugSettingsPanelComponent extends PageComponent implements @Input() allEnabledUntil = 0; @Input() maxDebugModeDuration: number; @Input() debugLimitsConfiguration: string; + @Input() additionalActionConfig: AdditionalDebugActionConfig; onFailuresControl = this.fb.control(false); debugAllControl = this.fb.control(false); diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 953d1d17c1..8de4a9f256 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -355,6 +355,86 @@ export class EventTableConfig extends EntityTableConfig { '48px') ); break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.columns[0].width = '160px'; + this.columns.push( + new EntityTableColumn('entityId', 'event.entity-id', '150px', + (entity) => entity.body.entityId, + () => ({padding: '0 12px 0 0'}), + false, + () => ({padding: '0 12px 0 0'}), + () => undefined, + false, + { + name: this.translate.instant('event.copy-entity-id'), + icon: 'content_paste', + style: { + padding: '4px', + 'font-size': '16px', + color: 'rgba(0,0,0,.87)' + }, + isEnabled: () => true, + onAction: ($event, entity) => entity.body.entityId, + type: CellActionDescriptorType.COPY_BUTTON + } + ), + new EntityTableColumn('messageId', 'event.message-id', '150px', + (entity) => entity.body.msgId ?? '', + () => ({padding: '0 12px 0 0'}), + false, + () => ({padding: '0 12px 0 0'}), + () => undefined, + false, + { + name: this.translate.instant('event.copy-message-id'), + icon: 'content_paste', + style: { + padding: '4px', + 'font-size': '16px', + color: 'rgba(0,0,0,.87)' + }, + isEnabled: () => true, + onAction: ($event, entity) => entity.body.msgId ?? '', + type: CellActionDescriptorType.COPY_BUTTON + } + ), + new EntityTableColumn('messageType', 'event.message-type', '150px', + (entity) => entity.body.msgType ?? '', + () => ({padding: '0 12px 0 0'}), + false + ), + new EntityActionTableColumn('arguments', 'event.arguments', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.arguments !== undefined, + onAction: ($event, entity) => this.showContent($event, entity.body.arguments, + 'event.arguments', ContentType.JSON, true) + }, + '100px' + ), + new EntityActionTableColumn('result', 'event.result', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.result !== undefined, + onAction: ($event, entity) => this.showContent($event, entity.body.result, + 'event.result', ContentType.JSON, true) + }, + '100px' + ), + new EntityActionTableColumn('error', 'event.error', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.error && entity.body.error.length > 0, + onAction: ($event, entity) => this.showContent($event, entity.body.error, + 'event.error') + }, + '100px' + ) + ); + break; } if (updateTableColumns) { this.getTable().columnsUpdated(true); @@ -446,6 +526,15 @@ export class EventTableConfig extends EntityTableConfig { {key: 'errorStr', title: 'event.error'} ); break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.filterColumns.push( + {key: 'entityId', title: 'event.entity-id'}, + {key: 'messageId', title: 'event.message-id'}, + {key: 'messageType', title: 'event.message-type'}, + {key: 'isError', title: 'event.error'}, + {key: 'errorStr', title: 'event.error'} + ); + break; } } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index f60ea15407..b7e35c5406 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -195,6 +195,9 @@ import { import { CalculatedFieldArgumentPanelComponent } from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { + CalculatedFieldDebugDialogComponent +} from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; @NgModule({ declarations: @@ -343,6 +346,7 @@ import { CalculatedFieldDialogComponent, CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, + CalculatedFieldDebugDialogComponent, ], imports: [ CommonModule, @@ -485,6 +489,7 @@ import { CalculatedFieldDialogComponent, CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, + CalculatedFieldDebugDialogComponent, ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 3a86b12018..31dca236a4 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -14,7 +14,12 @@ /// limitations under the License. /// -import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/entity.models'; +import { + AdditionalDebugActionConfig, + EntityDebugSettings, + HasTenantId, + HasVersion +} from '@shared/models/entity.models'; import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; import { EntityId } from '@shared/models/id/entity-id'; @@ -128,6 +133,13 @@ export interface CalculatedFieldDialogData { debugLimitsConfiguration: string; tenantId: string; entityName?: string; + additionalDebugActionConfig: AdditionalDebugActionConfig; +} + +export interface CalculatedFieldDebugDialogData { + id?: CalculatedFieldId; + entityId: EntityId; + tenantId: string; } export interface ArgumentEntityTypeParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 9934da65aa..db00955340 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -21,6 +21,7 @@ import { DeviceCredentialMQTTBasic } from '@shared/models/device.models'; import { Lwm2mSecurityConfigModels } from '@shared/models/lwm2m-security-config.models'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainMetaData } from '@shared/models/rule-chain.models'; +import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; export interface EntityInfo { name?: string; @@ -203,4 +204,12 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } +export type AdditionalDebugActionConfigData = CalculatedFieldDebugDialogData; + +export interface AdditionalDebugActionConfig { + action?: (data?: AdditionalDebugActionConfigData) => void; + title: string; + data: AdditionalDebugActionConfigData; +} + export type VersionedEntity = EntityInfoData & HasVersion | RuleChainMetaData; diff --git a/ui-ngx/src/app/shared/models/event.models.ts b/ui-ngx/src/app/shared/models/event.models.ts index a1abb8d15c..8a5009f758 100644 --- a/ui-ngx/src/app/shared/models/event.models.ts +++ b/ui-ngx/src/app/shared/models/event.models.ts @@ -29,7 +29,8 @@ export enum EventType { export enum DebugEventType { DEBUG_RULE_NODE = 'DEBUG_RULE_NODE', - DEBUG_RULE_CHAIN = 'DEBUG_RULE_CHAIN' + DEBUG_RULE_CHAIN = 'DEBUG_RULE_CHAIN', + DEBUG_CALCULATED_FIELD = 'DEBUG_CALCULATED_FIELD' } export const eventTypeTranslations = new Map( @@ -39,6 +40,7 @@ export const eventTypeTranslations = new Map [EventType.STATS, 'event.type-stats'], [DebugEventType.DEBUG_RULE_NODE, 'event.type-debug-rule-node'], [DebugEventType.DEBUG_RULE_CHAIN, 'event.type-debug-rule-chain'], + [DebugEventType.DEBUG_CALCULATED_FIELD, 'event.type-debug-calculated-field'], ] ); @@ -80,7 +82,7 @@ export interface DebugRuleChainEventBody extends BaseEventBody { error?: string; } -export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody; +export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody & CalculatedFieldEventBody; export interface Event extends BaseData { tenantId: TenantId; @@ -90,6 +92,16 @@ export interface Event extends BaseData { body: EventBody; } +export interface CalculatedFieldEventBody extends BaseFilterEventBody { + calculatedFieldId: string; + entityId: string; + entityType: EntityType; + arguments: string, + result: string, + msgId: string; + msgType: string; +} + export interface BaseFilterEventBody { server?: string; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 469bc9a00f..9bdf533fef 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1014,6 +1014,7 @@ "script": "Script" }, "arguments": "Arguments", + "debugging": "Calculated field debugging", "argument-name": "Argument name", "datasource": "Datasource", "add-argument": "Add argument", @@ -1026,6 +1027,7 @@ "argument-customer": "Customer", "argument-tenant": "Current tenant", "argument-type": "Argument type", + "see-debug-events": "See debug events", "attribute": "Attribute", "timeseries-key": "Time series key", "device-name": "Device name", @@ -2710,6 +2712,9 @@ "type-stats": "Statistics", "type-debug-rule-node": "Debug", "type-debug-rule-chain": "Debug", + "type-debug-calculated-field": "Debug", + "arguments": "Arguments", + "result": "Result", "no-events-prompt": "No events found", "error": "Error", "alarm": "Alarm", From 7ad8913b16b6d2ac0b99e57c0a7fa1ea520b4f79 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 15:52:29 +0200 Subject: [PATCH 200/416] refactoring --- .../debug-dialog/calculated-field-debug-dialog.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 1b61a9da4a..715cf0572d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -32,7 +32,7 @@ [disabledEventTypes]="[EventType.LC_EVENT, EventType.ERROR, EventType.STATS]" [defaultEventType]="DebugEventType.DEBUG_CALCULATED_FIELD" [active]="true" - [entityId]="data?.id" + [entityId]="data.id" [functionTestButtonLabel]="'common.test-function' | translate" /> From c8b7b1c3a243ef021080014cc54f7a83d5ca44db Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 15:53:55 +0200 Subject: [PATCH 201/416] refactoring --- .../app/modules/home/components/event/event-table-config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 8de4a9f256..da391d3ecb 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -379,7 +379,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '150px', - (entity) => entity.body.msgId ?? '', + (entity) => entity.body.msgId, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -394,12 +394,12 @@ export class EventTableConfig extends EntityTableConfig { color: 'rgba(0,0,0,.87)' }, isEnabled: () => true, - onAction: ($event, entity) => entity.body.msgId ?? '', + onAction: ($event, entity) => entity.body.msgId, type: CellActionDescriptorType.COPY_BUTTON } ), new EntityTableColumn('messageType', 'event.message-type', '150px', - (entity) => entity.body.msgType ?? '', + (entity) => entity.body.msgType, () => ({padding: '0 12px 0 0'}), false ), From 5354eaa80b7ffceece43aa7ea3d8d0e52e875aa1 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 16:10:26 +0200 Subject: [PATCH 202/416] Adjusted sizing --- .../calculated-field-debug-dialog.component.html | 2 +- .../home/components/event/event-table-config.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 715cf0572d..25f2a11f16 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index da391d3ecb..ae98560969 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -358,7 +358,7 @@ export class EventTableConfig extends EntityTableConfig { case DebugEventType.DEBUG_CALCULATED_FIELD: this.columns[0].width = '160px'; this.columns.push( - new EntityTableColumn('entityId', 'event.entity-id', '150px', + new EntityTableColumn('entityId', 'event.entity-id', '85px', (entity) => entity.body.entityId, () => ({padding: '0 12px 0 0'}), false, @@ -378,7 +378,7 @@ export class EventTableConfig extends EntityTableConfig { type: CellActionDescriptorType.COPY_BUTTON } ), - new EntityTableColumn('messageId', 'event.message-id', '150px', + new EntityTableColumn('messageId', 'event.message-id', '85px', (entity) => entity.body.msgId, () => ({padding: '0 12px 0 0'}), false, @@ -398,7 +398,7 @@ export class EventTableConfig extends EntityTableConfig { type: CellActionDescriptorType.COPY_BUTTON } ), - new EntityTableColumn('messageType', 'event.message-type', '150px', + new EntityTableColumn('messageType', 'event.message-type', '100px', (entity) => entity.body.msgType, () => ({padding: '0 12px 0 0'}), false @@ -411,7 +411,7 @@ export class EventTableConfig extends EntityTableConfig { onAction: ($event, entity) => this.showContent($event, entity.body.arguments, 'event.arguments', ContentType.JSON, true) }, - '100px' + '48px' ), new EntityActionTableColumn('result', 'event.result', { @@ -421,7 +421,7 @@ export class EventTableConfig extends EntityTableConfig { onAction: ($event, entity) => this.showContent($event, entity.body.result, 'event.result', ContentType.JSON, true) }, - '100px' + '48px' ), new EntityActionTableColumn('error', 'event.error', { @@ -431,7 +431,7 @@ export class EventTableConfig extends EntityTableConfig { onAction: ($event, entity) => this.showContent($event, entity.body.error, 'event.error') }, - '100px' + '48px' ) ); break; From 7fcd948071ffbefb0ff654758e58346a05150546 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Feb 2025 16:11:58 +0200 Subject: [PATCH 203/416] fixed error when no telemetry in db --- .../cf/DefaultCalculatedFieldCache.java | 3 ++- ...efaultCalculatedFieldExecutionService.java | 5 ++++- .../ctx/state/BaseCalculatedFieldState.java | 2 +- .../cf/ctx/state/RocksDBStateService.java | 19 +++++++++++++++---- .../ctx/state/SingleValueArgumentEntry.java | 9 ++++++++- .../state/SingleValueArgumentEntryTest.java | 4 ++++ .../server/common/data/event/EventFilter.java | 3 ++- 7 files changed, 36 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 9688f35fef..8fffa0029c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -32,8 +32,8 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.Collections; @@ -117,6 +117,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); + ctx.init(); calculatedFieldsCtx.put(calculatedFieldId, ctx); log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 103997f0c7..43e85d87e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,6 +39,7 @@ import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.OutputType; @@ -68,7 +69,6 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; @@ -447,6 +447,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); + if (StringUtils.isBlank(defaultValue)) { + return new StringDataEntry(key, null); + } if (NumberUtils.isParsable(defaultValue)) { return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 86d83a1f70..f1ce8038c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -52,7 +52,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { ArgumentEntry newEntry = entry.getValue(); ArgumentEntry existingEntry = arguments.get(key); - if (existingEntry == null) { + if (existingEntry == null || existingEntry == SingleValueArgumentEntry.EMPTY || existingEntry == TsRollingArgumentEntry.EMPTY) { validateNewEntry(newEntry); arguments.put(key, newEntry); stateUpdated = true; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 8a6a5c9cb7..b2e33e1705 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -63,7 +63,7 @@ public class RocksDBStateService implements CalculatedFieldStateService { CalculatedFieldStateProto stateProto = toProto(stateId, state); long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { - rocksDBService.put(toProto(stateId), toProto(stateId, state)); + rocksDBService.put(toProto(stateId), stateProto); } callback.onSuccess(); } @@ -111,8 +111,11 @@ public class RocksDBStateService implements CalculatedFieldStateService { private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() - .setArgName(argName) - .setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + .setArgName(argName); + + if (entry != SingleValueArgumentEntry.EMPTY) { + builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + } Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); @@ -122,7 +125,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { private TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { TsValueListProto.Builder builder = TsValueListProto.newBuilder().setKey(argName); - entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); + if (entry != TsRollingArgumentEntry.EMPTY) { + entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); + } return builder.build(); } @@ -151,6 +156,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { } private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { + if (!proto.hasValue()) { + return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY; + } TsValueProto tsValueProto = proto.getValue(); long ts = tsValueProto.getTs(); BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); @@ -158,6 +166,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { } private TsRollingArgumentEntry fromRollingArgumentProto(TsValueListProto proto) { + if (proto.getTsValueCount() <= 0) { + return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY; + } TreeMap tsRecords = new TreeMap<>(); proto.getTsValueList().forEach(tsValueProto -> { BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 0832e53e5e..d7e5ddd017 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -94,10 +94,17 @@ public class SingleValueArgumentEntry implements ArgumentEntry { Long newVersion = singleValueEntry.getVersion(); if (newVersion == null || this.version == null || newVersion > this.version) { this.ts = singleValueEntry.getTs(); - this.kvEntryValue = singleValueEntry.getKvEntryValue(); this.version = newVersion; + + // TODO: should we persist updated ts and version values? + BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); + if (this.kvEntryValue.getValue().equals(newValue.getValue())) { + return false; + } + this.kvEntryValue = singleValueEntry.getKvEntryValue(); return true; } + } else { throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 203d7b3d71..13651e852d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -69,4 +69,8 @@ public class SingleValueArgumentEntryTest { assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 234L))).isFalse(); } + @Test + void testUpdateEntryWhenValueWasNotChanged() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 237L))).isFalse(); + } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java index 6d2a110cf7..4c9791e3fb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java @@ -29,7 +29,8 @@ import io.swagger.v3.oas.annotations.media.Schema; @JsonSubTypes.Type(value = RuleChainDebugEventFilter.class, name = "DEBUG_RULE_CHAIN"), @JsonSubTypes.Type(value = ErrorEventFilter.class, name = "ERROR"), @JsonSubTypes.Type(value = LifeCycleEventFilter.class, name = "LC_EVENT"), - @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS") + @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS"), + @JsonSubTypes.Type(value = CalculatedFieldDebugEventFilter.class, name = "DEBUG_CALCULATED_FIELD") }) public interface EventFilter { From 908be4e1631c39e0aeac9f2498c05e2560228d57 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 6 Feb 2025 16:51:48 +0200 Subject: [PATCH 204/416] Update supported upgrade versions, cleanup --- .../DefaultDatabaseSchemaSettingsService.java | 30 ++----------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java index f1d9259e91..b414bfb13a 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java @@ -17,8 +17,6 @@ package org.thingsboard.server.service.install; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.info.BuildProperties; import org.springframework.context.annotation.Profile; import org.springframework.jdbc.core.JdbcTemplate; @@ -36,24 +34,16 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti private static final String CURRENT_PRODUCT = "CE"; // This list should include all versions which are compatible for the upgrade. // The compatibility cycle usually breaks when we have some scripts written in Java that may not work after new release. - private static final List SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("3.8.0", "3.8.1"); + private static final List SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("3.9.0"); private final BuildProperties buildProperties; private final JdbcTemplate jdbcTemplate; - @Value("${install.upgrade.from_version:}") - private String upgradeFromVersion; - private String packageSchemaVersion; private String schemaVersionFromDb; @Override public void validateSchemaSettings() { - //TODO: remove after release (3.9.0) - createProductIfNotExists(); - - String dbSchemaVersion = getDbSchemaVersion(); - if (DefaultDataUpdateService.getEnv("SKIP_SCHEMA_VERSION_CHECK", false)) { log.info("Skipped DB schema version check due to SKIP_SCHEMA_VERSION_CHECK set to 'true'."); return; @@ -64,6 +54,7 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti onSchemaSettingsError(String.format("Upgrade failed: can't upgrade ThingsBoard %s database using ThingsBoard %s.", product, CURRENT_PRODUCT)); } + String dbSchemaVersion = getDbSchemaVersion(); if (dbSchemaVersion.equals(getPackageSchemaVersion())) { onSchemaSettingsError("Upgrade failed: database already upgraded to current version. You can set SKIP_SCHEMA_VERSION_CHECK to 'true' if force re-upgrade needed."); } @@ -75,14 +66,6 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti } } - @Deprecated(forRemoval = true, since = "3.9.0") - private void createProductIfNotExists() { - boolean isCommunityEdition = jdbcTemplate.queryForList( - "SELECT 1 FROM information_schema.tables WHERE table_name = 'integration'", Integer.class).isEmpty(); - String product = isCommunityEdition ? "CE" : "PE"; - jdbcTemplate.execute("ALTER TABLE tb_schema_settings ADD COLUMN IF NOT EXISTS product varchar(2) DEFAULT '" + product + "'"); - } - @Override public void createSchemaSettings() { Long schemaVersion = getSchemaVersionFromDb(); @@ -107,15 +90,6 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti @Override public String getDbSchemaVersion() { if (schemaVersionFromDb == null) { - if (StringUtils.isNotBlank(upgradeFromVersion)) { - /* - * TODO - Remove after the release of 3.9.0: - * This a temporary workaround due to the issue that schema version in the - * tb_schema_settings was set as 3.6.4 during the install of 3.8.1. - * */ - schemaVersionFromDb = upgradeFromVersion; - return schemaVersionFromDb; - } Long version = getSchemaVersionFromDb(); if (version == null) { onSchemaSettingsError("Upgrade failed: the database schema version is missing."); From ff647aedc7c95c205fd4d27b52867b140504f2d4 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:01:33 +0200 Subject: [PATCH 205/416] Improved context passing --- .../calculated-fields-table-config.ts | 17 +++++++++-------- .../dialog/calculated-field-dialog.component.ts | 5 +---- ui-ngx/src/app/shared/models/entity.models.ts | 6 +----- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index c5e455583d..026c249159 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -42,6 +42,7 @@ import { } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, CalculatedFieldDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; +import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -52,11 +53,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, id), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -140,7 +137,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog({...this.additionalDebugActionConfig.data, id }), + action: () => this.openDebugDialog(id) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -206,11 +203,15 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data + data: { + tenantId: this.tenantId, + entityId: this.entityId, + id + } }) .afterClosed() .subscribe(); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 3575223764..d5bf243430 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -67,10 +67,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.data.additionalDebugActionConfig.action({ - ...this.data.additionalDebugActionConfig.data, - id: this.data.value.id, - }), + action: () => this.data.additionalDebugActionConfig.action(this.data.value.id) } : null; readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index db00955340..b9d2402850 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -21,7 +21,6 @@ import { DeviceCredentialMQTTBasic } from '@shared/models/device.models'; import { Lwm2mSecurityConfigModels } from '@shared/models/lwm2m-security-config.models'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainMetaData } from '@shared/models/rule-chain.models'; -import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; export interface EntityInfo { name?: string; @@ -204,12 +203,9 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } -export type AdditionalDebugActionConfigData = CalculatedFieldDebugDialogData; - export interface AdditionalDebugActionConfig { - action?: (data?: AdditionalDebugActionConfigData) => void; + action?: (id?: EntityId) => void; title: string; - data: AdditionalDebugActionConfigData; } export type VersionedEntity = EntityInfoData & HasVersion | RuleChainMetaData; From ebab88ac6a06d04d3141c53107b927cd1257bcef Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Feb 2025 17:06:21 +0200 Subject: [PATCH 206/416] added logs --- ...CalculatedFieldEntityMessageProcessor.java | 7 +++++ ...alculatedFieldManagerMessageProcessor.java | 28 +++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 309dfde2e9..544dea44c9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -83,12 +83,15 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldStateRestoreMsg msg) { + log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), msg.getId().cfId()); states.put(msg.getId().cfId(), msg.getState()); } public void process(EntityInitCalculatedFieldMsg msg) { + log.info("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); var cfCtx = msg.getCtx(); if (msg.isForceReinit()) { + log.info("Force reinitialization of CF: [{}].", cfCtx.getCfId()); states.remove(cfCtx.getCfId()); } var cfState = getOrInitState(cfCtx); @@ -96,6 +99,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldEntityDeleteMsg msg) { + log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); states.forEach((cfId, state) -> cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); @@ -110,6 +114,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(EntityCalculatedFieldTelemetryMsg msg) { + log.info("[{}] Processing CF telemetry msg.", msg.getEntityId()); var proto = msg.getProto(); var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); @@ -124,6 +129,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) { + log.info("[{}] Processing CF link telemetry msg.", msg.getEntityId()); var proto = msg.getProto(); var ctx = msg.getCtx(); var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); @@ -169,6 +175,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) { if (newArgValues.isEmpty()) { + log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); callback.onSuccess(CALLBACKS_PER_CF); } CalculatedFieldState state = getOrInitState(ctx); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index c219544a91..7b03ed2938 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -91,11 +91,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onFieldInitMsg(CalculatedFieldInitMsg msg) { + log.info("[{}] Processing CF init message.", msg.getCf().getId()); var cf = msg.getCf(); var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { + log.debug("[{}] Failed to initialize CF context.", cf.getId(), e); if (DebugModeUtil.isDebugAllAvailable(cf)) { systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); } @@ -108,6 +110,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { + log.info("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId()); var link = msg.getLink(); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) @@ -121,6 +124,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (calculatedField != null) { msg.getState().setRequiredArguments(calculatedField.getArgNames()); + log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); getOrCreateActor(msg.getId().entityId()).tell(msg); } else { cfExecService.deleteStateFromStorage(msg.getId(), msg.getCallback()); @@ -128,6 +132,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) { + log.info("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); var event = msg.getData().getEvent(); switch (entityType) { @@ -207,6 +212,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { cfEntityCache.evict(tenantId, msg.getEntityId()); + log.info("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); } @@ -225,6 +231,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware try { cfCtx.init(); } catch (Exception e) { + log.debug("[{}] Failed to initialize CF context.", cf.getId(), e); if (DebugModeUtil.isDebugAllAvailable(cf)) { systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); } @@ -251,6 +258,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + newCfCtx.init(); + } catch (Exception e) { + log.debug("[{}] Failed to initialize CF context.", newCf.getId(), e); + if (DebugModeUtil.isDebugAllAvailable(newCf)) { + systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); @@ -318,12 +333,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); + log.info("Received telemetry msg from entity [{}]", entityId); // 2 = 1 for CF processing + 1 for links processing MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); // process all cfs related to entity, or it's profile; var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { + log.info("Pushing telemetry msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); } else { callback.onSuccess(); @@ -340,6 +357,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { EntityId sourceEntityId = msg.getEntityId(); + log.info("Received linked telemetry msg from entity [{}]", sourceEntityId); var proto = msg.getProto(); var linksList = proto.getLinksList(); for (var linkProto : linksList) { @@ -353,12 +371,15 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!entityIds.isEmpty()) { MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); - entityIds.forEach(entityId -> getOrCreateActor(entityId).tell(newMsg)); + entityIds.forEach(entityId -> { + log.info("Pushing linked telemetry msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(newMsg); + }); } else { msg.getCallback().onSuccess(); } } else { - // push the message to specific entity; + log.info("Pushing linked telemetry msg to specific actor [{}]", targetEntityId); var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); getOrCreateActor(targetEntityId).tell(newMsg); } @@ -423,10 +444,12 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { + log.info("Pushing delete CF msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); } private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { + log.info("Pushing entity init CF msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); } @@ -460,5 +483,4 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).remove(link)); } - } From daf0b78ac5dc93e7faf5ca03a0741452975dcdf4 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 6 Feb 2025 17:07:13 +0200 Subject: [PATCH 207/416] Optimize imports --- .../service/install/DefaultDatabaseSchemaSettingsService.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java index 229e21a96d..3b21685d22 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java @@ -17,9 +17,6 @@ package org.thingsboard.server.service.install; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.info.BuildProperties; import org.springframework.context.annotation.Profile; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; From dfb22314e4783af3cdacc0d8f490ffe6741e0474 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:13:04 +0200 Subject: [PATCH 208/416] Improved styling --- ...lculated-field-debug-dialog.component.html | 4 ++-- ...lculated-field-debug-dialog.component.scss | 24 +++++++++++++++++++ ...calculated-field-debug-dialog.component.ts | 1 + 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 25f2a11f16..6166765bc2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

@@ -25,7 +25,7 @@ close -
+
implements AfterViewInit { From 0c226afbc8177968836974ecfb0572cffd59ea78 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:13:45 +0200 Subject: [PATCH 209/416] class order --- .../debug-dialog/calculated-field-debug-dialog.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 6166765bc2..3d92ba5454 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

@@ -25,7 +25,7 @@ close -
+
Date: Thu, 6 Feb 2025 17:14:16 +0200 Subject: [PATCH 210/416] Improved styling --- .../debug-dialog/calculated-field-debug-dialog.component.html | 2 +- .../debug-dialog/calculated-field-debug-dialog.component.scss | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 3d92ba5454..91da675fea 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss index 19bf072b11..1e33c9371e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -16,6 +16,7 @@ :host { .debug-dialog-container { height: 77vh; + min-width: 90vw; .debug-dialog-content { border-radius: 0; From c167c6969fa82b8e83a17c6fe7627e25c9786e25 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:17:10 +0200 Subject: [PATCH 211/416] Improved styling --- .../debug-dialog/calculated-field-debug-dialog.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss index 1e33c9371e..23aa4070d7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -16,7 +16,7 @@ :host { .debug-dialog-container { height: 77vh; - min-width: 90vw; + min-width: 80vw; .debug-dialog-content { border-radius: 0; From bee6375572e42f6f39e6db4fb45e0647a38efd19 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:31:59 +0200 Subject: [PATCH 212/416] Improved styling --- .../modules/home/components/event/event-table-config.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index ae98560969..a937e42bb8 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -356,10 +356,11 @@ export class EventTableConfig extends EntityTableConfig { ); break; case DebugEventType.DEBUG_CALCULATED_FIELD: - this.columns[0].width = '160px'; + this.columns[0].width = '80px'; + this.columns[1].width = '20%'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', - (entity) => entity.body.entityId, + (entity) => `${entity.body.entityId.substring(0, 6)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -379,7 +380,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '85px', - (entity) => entity.body.msgId, + (entity) => `${entity.body.msgId?.substring(0, 6)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), From c582740175ce95814cc80e458505f2b181c29acc Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 09:56:42 +0200 Subject: [PATCH 213/416] added endpoint to test script --- .../controller/CalculatedFieldController.java | 84 +++++++++++++++++++ .../state/CalculatedFieldScriptEngine.java | 3 + .../CalculatedFieldTbelScriptEngine.java | 7 ++ 3 files changed, 94 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 96a97aeeff..435e46069d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.controller; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; import lombok.RequiredArgsConstructor; @@ -29,6 +32,8 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -38,16 +43,27 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldScriptEngine; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngine; import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -59,9 +75,29 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI public class CalculatedFieldController extends BaseController { private final TbCalculatedFieldService tbCalculatedFieldService; + private final TbelInvokeService tbelInvokeService; public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; + public static final int TIMEOUT = 20; + + private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"expression\": \"var temp = 0; foreach(element: temperature.entrySet()) { temp += element.getValue(); } var avgTemperature = temp / temperature.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity; return { \\\"adjustedTemperature\\\": adjustedTemperature };\",\n" + + " \"argNames\": [\"temperature\", \"humidity\"],\n" + + " \"arguments\": {\n" + + " \"temperature\": {\n" + + " \"14327856345\": 22.4,\n" + + " \"14327857298\": 21.9,\n" + + " \"14327857510\": 22.0\n" + + " },\n" + + " \"humidity\": 42\n" + + " }\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + "\n\n Expected result JSON contains \"output\" and \"error\"."; + @ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + "The newly created Calculated Field Id will be present in the response. " + @@ -128,4 +164,52 @@ public class CalculatedFieldController extends BaseController { tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); } + @ApiOperation(value = "Test Script expression", + notes = TEST_SCRIPT_EXPRESSION + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/testScript", method = RequestMethod.POST) + @ResponseBody + public JsonNode testScript( + @io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") + @RequestBody JsonNode inputParams) { + String expression = inputParams.get("expression").asText(); + String[] argNames = JacksonUtil.treeToValue(inputParams.get("argNames"), String[].class); + Map arguments = Objects.requireNonNullElse( + JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference>() { + }), + Collections.emptyMap() + ); + + String output = ""; + String errorText = ""; + + try { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + CalculatedFieldScriptEngine calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( + getTenantId(), + tbelInvokeService, + expression, + argNames + ); + + Object[] args = Arrays.stream(argNames) + .map(arguments::get) + .toArray(); + + JsonNode json = calculatedFieldScriptEngine.executeJsonAsync(args).get(TIMEOUT, TimeUnit.SECONDS); + output = JacksonUtil.toString(json); + } catch (Exception e) { + log.error("Error evaluating expression", e); + errorText = e.getMessage(); + } + + ObjectNode result = JacksonUtil.newObjectNode(); + result.put("output", output); + result.put("error", errorText); + return result; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 779f52c5d6..6bea6ce705 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import java.util.Map; @@ -25,6 +26,8 @@ public interface CalculatedFieldScriptEngine { ListenableFuture> executeToMapAsync(Object[] args); + ListenableFuture executeJsonAsync(Object[] args); + ListenableFuture> executeToMapTransform(Object result); void destroy(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 7ac032573f..9e05e05970 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -74,6 +76,11 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng return Futures.transformAsync(executeScriptAsync(args), this::executeToMapTransform, MoreExecutors.directExecutor()); } + @Override + public ListenableFuture executeJsonAsync(Object[] args) { + return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor()); + } + @Override public ListenableFuture> executeToMapTransform(Object result) { if (result instanceof Map) { From 12a8c070a674b35b4dffc8ae5599e6acc6ed590f Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 7 Feb 2025 11:45:15 +0200 Subject: [PATCH 214/416] Change wording from persistence to processing on backend --- .../rule_chains/edge_root_rule_chain.json | 2 +- .../device_profile/rule_chain_template.json | 2 +- .../tenant/rule_chains/root_rule_chain.json | 2 +- .../main/data/upgrade/basic/schema_update.sql | 4 +- .../src/main/resources/root_rule_chain.json | 6 +- .../resources/MqttRuleNodeTestMetadata.json | 2 +- .../engine/telemetry/TbMsgTimeseriesNode.java | 52 ++++++------- .../TbMsgTimeseriesNodeConfiguration.java | 28 +++---- ...ava => DeduplicateProcessingStrategy.java} | 6 +- ... => OnEveryMessageProcessingStrategy.java} | 10 +-- ...eStrategy.java => ProcessingStrategy.java} | 22 +++--- ...ategy.java => SkipProcessingStrategy.java} | 10 +-- .../telemetry/TbMsgTimeseriesNodeTest.java | 72 +++++++++--------- ...=> DeduplicateProcessingStrategyTest.java} | 76 +++++++++---------- ...OnEveryMessageProcessingStrategyTest.java} | 6 +- ...yTest.java => ProcessingStrategyTest.java} | 14 ++-- ...t.java => SkipProcessingStrategyTest.java} | 6 +- 17 files changed, 160 insertions(+), 160 deletions(-) rename rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/{DeduplicatePersistenceStrategy.java => DeduplicateProcessingStrategy.java} (94%) rename rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/{OnEveryMessagePersistenceStrategy.java => OnEveryMessageProcessingStrategy.java} (69%) rename rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/{PersistenceStrategy.java => ProcessingStrategy.java} (53%) rename rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/{SkipPersistenceStrategy.java => SkipProcessingStrategy.java} (72%) rename rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/{DeduplicatePersistenceStrategyTest.java => DeduplicateProcessingStrategyTest.java} (75%) rename rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/{OnEveryMessagePersistenceStrategyTest.java => OnEveryMessageProcessingStrategyTest.java} (89%) rename rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/{PersistenceStrategyTest.java => ProcessingStrategyTest.java} (76%) rename rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/{SkipPersistenceStrategyTest.java => SkipProcessingStrategyTest.java} (90%) diff --git a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json index e663ff779d..6701b59e0e 100644 --- a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json +++ b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json @@ -37,7 +37,7 @@ "configuration": { "defaultTTL": 0, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ON_EVERY_MESSAGE" } }, diff --git a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json index 325e01003f..0f2473cde6 100644 --- a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json +++ b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json @@ -23,7 +23,7 @@ "configuration": { "defaultTTL": 0, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ON_EVERY_MESSAGE" } } diff --git a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json index 4dc202d740..8efda98c5b 100644 --- a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json @@ -22,7 +22,7 @@ "configuration": { "defaultTTL": 0, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ON_EVERY_MESSAGE" } } diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 31833112ad..a1ef460854 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -225,7 +225,7 @@ DO $$ SET configuration = ( (configuration::jsonb - 'skipLatestPersistence') || jsonb_build_object( - 'persistenceSettings', jsonb_build_object( + 'processingSettings', jsonb_build_object( 'type', 'ADVANCED', 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), 'latest', jsonb_build_object('type', 'SKIP'), @@ -242,7 +242,7 @@ DO $$ SET configuration = ( (configuration::jsonb - 'skipLatestPersistence') || jsonb_build_object( - 'persistenceSettings', jsonb_build_object( + 'processingSettings', jsonb_build_object( 'type', 'ON_EVERY_MESSAGE' ) ) diff --git a/monitoring/src/main/resources/root_rule_chain.json b/monitoring/src/main/resources/root_rule_chain.json index eda3e44e1b..46bdc72d9f 100644 --- a/monitoring/src/main/resources/root_rule_chain.json +++ b/monitoring/src/main/resources/root_rule_chain.json @@ -25,7 +25,7 @@ "configuration": { "defaultTTL": 0, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ON_EVERY_MESSAGE" } }, @@ -281,7 +281,7 @@ "configuration": { "defaultTTL": 0, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ON_EVERY_MESSAGE" } }, @@ -319,7 +319,7 @@ "configuration": { "defaultTTL": 180, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ON_EVERY_MESSAGE" } }, diff --git a/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json b/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json index c495e590cd..c09da6aebb 100644 --- a/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json +++ b/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json @@ -40,7 +40,7 @@ "configuration": { "defaultTTL": 0, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ON_EVERY_MESSAGE" } }, diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index b98b1d205e..1b3acb6dae 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -27,7 +27,7 @@ import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.rule.engine.telemetry.strategy.PersistenceStrategy; +import org.thingsboard.rule.engine.telemetry.strategy.ProcessingStrategy; import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TenantProfile; @@ -45,11 +45,11 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Deduplicate; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.OnEveryMessage; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.WebSocketsOnly; +import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings; +import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced; +import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Deduplicate; +import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.OnEveryMessage; +import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.WebSocketsOnly; import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_REQUEST; @Slf4j @@ -58,7 +58,7 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE name = "save time series", configClazz = TbMsgTimeseriesNodeConfiguration.class, nodeDescription = """ - Saves time series data with a configurable TTL and according to configured persistence strategies. + Saves time series data with a configurable TTL and according to configured processing strategies. """, nodeDetails = """ Node performs three actions: @@ -68,14 +68,14 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE
  • WebSockets: notify WebSockets subscriptions about time series data updates.
    • - For each action, three persistence strategies are available: + For each action, three processing strategies are available:
    • On every message: perform the action for every message.
    • Deduplicate: perform the action only for the first message from a particular originator within a configurable interval.
    • Skip: never perform the action.
    - Persistence strategies are configured using persistence settings, which support two modes: + Processing strategies are configured using processing settings, which support two modes:
    • Basic
        @@ -110,7 +110,7 @@ public class TbMsgTimeseriesNode implements TbNode { private TbContext ctx; private long tenantProfileDefaultStorageTtl; - private PersistenceSettings persistenceSettings; + private ProcessingSettings processingSettings; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -118,7 +118,7 @@ public class TbMsgTimeseriesNode implements TbNode { this.ctx = ctx; ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); - persistenceSettings = config.getPersistenceSettings(); + processingSettings = config.getProcessingSettings(); } private void onTenantProfileUpdate(TenantProfile tenantProfile) { @@ -175,25 +175,25 @@ public class TbMsgTimeseriesNode implements TbNode { } private TimeseriesSaveRequest.Strategy determineSaveStrategy(long ts, UUID originatorUuid) { - if (persistenceSettings instanceof OnEveryMessage) { + if (processingSettings instanceof OnEveryMessage) { return TimeseriesSaveRequest.Strategy.SAVE_ALL; } - if (persistenceSettings instanceof WebSocketsOnly) { + if (processingSettings instanceof WebSocketsOnly) { return TimeseriesSaveRequest.Strategy.WS_ONLY; } - if (persistenceSettings instanceof Deduplicate deduplicate) { - boolean isFirstMsgInInterval = deduplicate.getPersistenceStrategy().shouldPersist(ts, originatorUuid); + if (processingSettings instanceof Deduplicate deduplicate) { + boolean isFirstMsgInInterval = deduplicate.getProcessingStrategy().shouldProcess(ts, originatorUuid); return isFirstMsgInInterval ? TimeseriesSaveRequest.Strategy.SAVE_ALL : TimeseriesSaveRequest.Strategy.SKIP_ALL; } - if (persistenceSettings instanceof Advanced advanced) { + if (processingSettings instanceof Advanced advanced) { return new TimeseriesSaveRequest.Strategy( - advanced.timeseries().shouldPersist(ts, originatorUuid), - advanced.latest().shouldPersist(ts, originatorUuid), - advanced.webSockets().shouldPersist(ts, originatorUuid) + advanced.timeseries().shouldProcess(ts, originatorUuid), + advanced.latest().shouldProcess(ts, originatorUuid), + advanced.webSockets().shouldProcess(ts, originatorUuid) ); } // should not happen - throw new IllegalArgumentException("Unknown persistence settings type: " + persistenceSettings.getClass().getSimpleName()); + throw new IllegalArgumentException("Unknown processing settings type: " + processingSettings.getClass().getSimpleName()); } @Override @@ -209,14 +209,14 @@ public class TbMsgTimeseriesNode implements TbNode { hasChanges = true; JsonNode skipLatestPersistence = oldConfiguration.get("skipLatestPersistence"); if (skipLatestPersistence != null && "true".equals(skipLatestPersistence.asText())) { - var skipLatestPersistenceSettings = new Advanced( - PersistenceStrategy.onEveryMessage(), - PersistenceStrategy.skip(), - PersistenceStrategy.onEveryMessage() + var skipLatestProcessingSettings = new Advanced( + ProcessingStrategy.onEveryMessage(), + ProcessingStrategy.skip(), + ProcessingStrategy.onEveryMessage() ); - ((ObjectNode) oldConfiguration).set("persistenceSettings", JacksonUtil.valueToTree(skipLatestPersistenceSettings)); + ((ObjectNode) oldConfiguration).set("processingSettings", JacksonUtil.valueToTree(skipLatestProcessingSettings)); } else { - ((ObjectNode) oldConfiguration).set("persistenceSettings", JacksonUtil.valueToTree(new OnEveryMessage())); + ((ObjectNode) oldConfiguration).set("processingSettings", JacksonUtil.valueToTree(new OnEveryMessage())); } ((ObjectNode) oldConfiguration).remove("skipLatestPersistence"); break; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java index a1386fee49..88524e3751 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java @@ -24,14 +24,14 @@ import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.Getter; import org.thingsboard.rule.engine.api.NodeConfiguration; -import org.thingsboard.rule.engine.telemetry.strategy.PersistenceStrategy; +import org.thingsboard.rule.engine.telemetry.strategy.ProcessingStrategy; import java.util.Objects; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Deduplicate; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.OnEveryMessage; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.WebSocketsOnly; +import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced; +import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Deduplicate; +import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.OnEveryMessage; +import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.WebSocketsOnly; @Data public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration { @@ -39,14 +39,14 @@ public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration> deduplicationCache; @JsonCreator - public DeduplicatePersistenceStrategy(@JsonProperty("deduplicationIntervalSecs") int deduplicationIntervalSecs) { + public DeduplicateProcessingStrategy(@JsonProperty("deduplicationIntervalSecs") int deduplicationIntervalSecs) { if (deduplicationIntervalSecs < MIN_DEDUPLICATION_INTERVAL_SECS || deduplicationIntervalSecs > MAX_DEDUPLICATION_INTERVAL_SECS) { throw new IllegalArgumentException("Deduplication interval must be at least " + MIN_DEDUPLICATION_INTERVAL_SECS + " second(s) " + "and at most " + MAX_DEDUPLICATION_INTERVAL_SECS + " second(s), was " + deduplicationIntervalSecs + " second(s)"); @@ -81,7 +81,7 @@ final class DeduplicatePersistenceStrategy implements PersistenceStrategy { } @Override - public boolean shouldPersist(long ts, UUID originatorUuid) { + public boolean shouldProcess(long ts, UUID originatorUuid) { long intervalNumber = ts / deduplicationIntervalMillis; return deduplicationCache.get(intervalNumber).add(originatorUuid); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessageProcessingStrategy.java similarity index 69% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategy.java rename to rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessageProcessingStrategy.java index 4fcb74dc33..9e8a87364a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategy.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessageProcessingStrategy.java @@ -19,19 +19,19 @@ import com.fasterxml.jackson.annotation.JsonCreator; import java.util.UUID; -final class OnEveryMessagePersistenceStrategy implements PersistenceStrategy { +final class OnEveryMessageProcessingStrategy implements ProcessingStrategy { - private static final OnEveryMessagePersistenceStrategy INSTANCE = new OnEveryMessagePersistenceStrategy(); + private static final OnEveryMessageProcessingStrategy INSTANCE = new OnEveryMessageProcessingStrategy(); - private OnEveryMessagePersistenceStrategy() {} + private OnEveryMessageProcessingStrategy() {} @JsonCreator - public static OnEveryMessagePersistenceStrategy getInstance() { + public static OnEveryMessageProcessingStrategy getInstance() { return INSTANCE; } @Override - public boolean shouldPersist(long ts, UUID originatorUuid) { + public boolean shouldProcess(long ts, UUID originatorUuid) { return true; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/ProcessingStrategy.java similarity index 53% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategy.java rename to rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/ProcessingStrategy.java index 453092117b..724727dc22 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategy.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/ProcessingStrategy.java @@ -26,24 +26,24 @@ import java.util.UUID; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = OnEveryMessagePersistenceStrategy.class, name = "ON_EVERY_MESSAGE"), - @JsonSubTypes.Type(value = DeduplicatePersistenceStrategy.class, name = "DEDUPLICATE"), - @JsonSubTypes.Type(value = SkipPersistenceStrategy.class, name = "SKIP") + @JsonSubTypes.Type(value = OnEveryMessageProcessingStrategy.class, name = "ON_EVERY_MESSAGE"), + @JsonSubTypes.Type(value = DeduplicateProcessingStrategy.class, name = "DEDUPLICATE"), + @JsonSubTypes.Type(value = SkipProcessingStrategy.class, name = "SKIP") }) -public sealed interface PersistenceStrategy permits OnEveryMessagePersistenceStrategy, DeduplicatePersistenceStrategy, SkipPersistenceStrategy { +public sealed interface ProcessingStrategy permits OnEveryMessageProcessingStrategy, DeduplicateProcessingStrategy, SkipProcessingStrategy { - static PersistenceStrategy onEveryMessage() { - return OnEveryMessagePersistenceStrategy.getInstance(); + static ProcessingStrategy onEveryMessage() { + return OnEveryMessageProcessingStrategy.getInstance(); } - static PersistenceStrategy deduplicate(int deduplicationIntervalSecs) { - return new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + static ProcessingStrategy deduplicate(int deduplicationIntervalSecs) { + return new DeduplicateProcessingStrategy(deduplicationIntervalSecs); } - static PersistenceStrategy skip() { - return SkipPersistenceStrategy.getInstance(); + static ProcessingStrategy skip() { + return SkipProcessingStrategy.getInstance(); } - boolean shouldPersist(long ts, UUID originatorUuid); + boolean shouldProcess(long ts, UUID originatorUuid); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/SkipProcessingStrategy.java similarity index 72% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategy.java rename to rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/SkipProcessingStrategy.java index c3d96e8ca7..66b966653d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategy.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/SkipProcessingStrategy.java @@ -19,19 +19,19 @@ import com.fasterxml.jackson.annotation.JsonCreator; import java.util.UUID; -final class SkipPersistenceStrategy implements PersistenceStrategy { +final class SkipProcessingStrategy implements ProcessingStrategy { - private static final SkipPersistenceStrategy INSTANCE = new SkipPersistenceStrategy(); + private static final SkipProcessingStrategy INSTANCE = new SkipProcessingStrategy(); - private SkipPersistenceStrategy() {} + private SkipProcessingStrategy() {} @JsonCreator - public static SkipPersistenceStrategy getInstance() { + public static SkipProcessingStrategy getInstance() { return INSTANCE; } @Override - public boolean shouldPersist(long ts, UUID originatorUuid) { + public boolean shouldProcess(long ts, UUID originatorUuid) { return false; } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index 02c19ed5fc..892e217414 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -34,7 +34,7 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; -import org.thingsboard.rule.engine.telemetry.strategy.PersistenceStrategy; +import org.thingsboard.rule.engine.telemetry.strategy.ProcessingStrategy; import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.DeviceId; @@ -110,7 +110,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @Test public void verifyDefaultConfig() { assertThat(config.getDefaultTTL()).isEqualTo(0L); - assertThat(config.getPersistenceSettings()).isInstanceOf(TbMsgTimeseriesNodeConfiguration.PersistenceSettings.OnEveryMessage.class); + assertThat(config.getProcessingSettings()).isInstanceOf(TbMsgTimeseriesNodeConfiguration.ProcessingSettings.OnEveryMessage.class); assertThat(config.isUseServerTs()).isFalse(); } @@ -124,14 +124,14 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenPersistenceSettingsAreNull_whenValidatingConstraints_thenThrowsException() { + public void givenProcessingSettingsAreNull_whenValidatingConstraints_thenThrowsException() { // GIVEN - config.setPersistenceSettings(null); + config.setProcessingSettings(null); // WHEN-THEN assertThatThrownBy(() -> ConstraintValidator.validateFields(config)) .isInstanceOf(DataValidationException.class) - .hasMessage("Validation error: persistenceSettings must not be null"); + .hasMessage("Validation error: processingSettings must not be null"); } @ParameterizedTest @@ -216,15 +216,15 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenSkipLatestPersistenceSettingsAndTtlFromConfig_whenOnMsg_thenSaveTimeseriesUsingTtlFromConfig() throws TbNodeException { + public void givenSkipLatestProcessingSettingsAndTtlFromConfig_whenOnMsg_thenSaveTimeseriesUsingTtlFromConfig() throws TbNodeException { // GIVEN config.setDefaultTTL(10L); - var timeseriesStrategy = PersistenceStrategy.onEveryMessage(); - var latestStrategy = PersistenceStrategy.skip(); - var webSockets = PersistenceStrategy.onEveryMessage(); - var persistenceSettings = new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced(timeseriesStrategy, latestStrategy, webSockets); - config.setPersistenceSettings(persistenceSettings); + var timeseriesStrategy = ProcessingStrategy.onEveryMessage(); + var latestStrategy = ProcessingStrategy.skip(); + var webSockets = ProcessingStrategy.onEveryMessage(); + var processingSettings = new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced(timeseriesStrategy, latestStrategy, webSockets); + config.setProcessingSettings(processingSettings); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -333,9 +333,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenOnEveryMessagePersistenceSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { + public void givenOnEveryMessageProcessingSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { // GIVEN - config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.OnEveryMessage()); + config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.OnEveryMessage()); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -368,9 +368,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenDeduplicatePersistenceSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistThisMessageOnlyFirstTime() throws TbNodeException { + public void givenDeduplicateProcessingSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistThisMessageOnlyFirstTime() throws TbNodeException { // GIVEN - config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Deduplicate(10)); + config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Deduplicate(10)); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -403,9 +403,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenWebsocketsOnlyPersistenceSettingsAndSameMessageTwoTimes_whenOnMsg_thenSendsOnlyWsUpdateTwoTimes() throws TbNodeException { + public void givenWebSocketsOnlyProcessingSettingsAndSameMessageTwoTimes_whenOnMsg_thenSendsOnlyWsUpdateTwoTimes() throws TbNodeException { // GIVEN - config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.WebSocketsOnly()); + config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.WebSocketsOnly()); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -438,12 +438,12 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenAdvancedPersistenceSettingsWithOnEveryMessageStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { + public void givenAdvancedProcessingSettingsWithOnEveryMessageStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { // GIVEN - config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced( - PersistenceStrategy.onEveryMessage(), - PersistenceStrategy.onEveryMessage(), - PersistenceStrategy.onEveryMessage() + config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced( + ProcessingStrategy.onEveryMessage(), + ProcessingStrategy.onEveryMessage(), + ProcessingStrategy.onEveryMessage() )); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -477,12 +477,12 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenAdvancedPersistenceSettingsWithDifferentDeduplicateStrategyForEachAction_whenOnMsg_thenEvaluatesStrategiesForEachActionsIndependently() throws TbNodeException { + public void givenAdvancedProcessingSettingsWithDifferentDeduplicateStrategyForEachAction_whenOnMsg_thenEvaluatesStrategiesForEachActionsIndependently() throws TbNodeException { // GIVEN - config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced( - PersistenceStrategy.deduplicate(1), - PersistenceStrategy.deduplicate(2), - PersistenceStrategy.deduplicate(3) + config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced( + ProcessingStrategy.deduplicate(1), + ProcessingStrategy.deduplicate(2), + ProcessingStrategy.deduplicate(3) )); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -528,12 +528,12 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenAdvancedPersistenceSettingsWithSkipStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenSkipsSameMessageTwoTimes() throws TbNodeException { + public void givenAdvancedProcessingSettingsWithSkipStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenSkipsSameMessageTwoTimes() throws TbNodeException { // GIVEN - config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced( - PersistenceStrategy.skip(), - PersistenceStrategy.skip(), - PersistenceStrategy.skip() + config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced( + ProcessingStrategy.skip(), + ProcessingStrategy.skip(), + ProcessingStrategy.skip() )); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -577,7 +577,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { { "defaultTTL": 0, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ON_EVERY_MESSAGE" } }"""), @@ -591,7 +591,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { { "defaultTTL": 0, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ON_EVERY_MESSAGE" } }"""), @@ -606,7 +606,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { { "defaultTTL": 0, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ON_EVERY_MESSAGE" } }"""), @@ -621,7 +621,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { { "defaultTTL": 0, "useServerTs": false, - "persistenceSettings": { + "processingSettings": { "type": "ADVANCED", "timeseries": { "type": "ON_EVERY_MESSAGE" diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicateProcessingStrategyTest.java similarity index 75% rename from rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java rename to rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicateProcessingStrategyTest.java index 1ae2b367a5..2c81ecd0a1 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicateProcessingStrategyTest.java @@ -28,27 +28,27 @@ import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -class DeduplicatePersistenceStrategyTest { +class DeduplicateProcessingStrategyTest { final int deduplicationIntervalSecs = 10; - DeduplicatePersistenceStrategy strategy; + DeduplicateProcessingStrategy strategy; @BeforeEach void setup() { - strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + strategy = new DeduplicateProcessingStrategy(deduplicationIntervalSecs); } @Test void shouldThrowWhenDeduplicationIntervalIsLessThanOneSecond() { - assertThatThrownBy(() -> new DeduplicatePersistenceStrategy(0)) + assertThatThrownBy(() -> new DeduplicateProcessingStrategy(0)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Deduplication interval must be at least 1 second(s) and at most 86400 second(s), was 0 second(s)"); } @Test void shouldThrowWhenDeduplicationIntervalIsMoreThan24Hours() { - assertThatThrownBy(() -> new DeduplicatePersistenceStrategy(86401)) + assertThatThrownBy(() -> new DeduplicateProcessingStrategy(86401)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Deduplication interval must be at least 1 second(s) and at most 86400 second(s), was 86401 second(s)"); } @@ -59,7 +59,7 @@ class DeduplicatePersistenceStrategyTest { int deduplicationIntervalSecs = 1; // min deduplication interval duration // WHEN - strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + strategy = new DeduplicateProcessingStrategy(deduplicationIntervalSecs); // THEN var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); @@ -76,7 +76,7 @@ class DeduplicatePersistenceStrategyTest { int deduplicationIntervalSecs = (int) Duration.ofHours(1L).toSeconds(); // max deduplication interval duration // WHEN - strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + strategy = new DeduplicateProcessingStrategy(deduplicationIntervalSecs); // THEN var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); @@ -93,7 +93,7 @@ class DeduplicatePersistenceStrategyTest { int deduplicationIntervalSecs = (int) Duration.ofDays(1L).toSeconds(); // max deduplication interval duration // WHEN - strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + strategy = new DeduplicateProcessingStrategy(deduplicationIntervalSecs); // THEN var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); @@ -110,7 +110,7 @@ class DeduplicatePersistenceStrategyTest { int deduplicationIntervalSecs = 1; // min deduplication interval duration // WHEN - strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + strategy = new DeduplicateProcessingStrategy(deduplicationIntervalSecs); // THEN var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); @@ -127,7 +127,7 @@ class DeduplicatePersistenceStrategyTest { int deduplicationIntervalSecs = (int) Duration.ofHours(1L).toSeconds(); // WHEN - strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + strategy = new DeduplicateProcessingStrategy(deduplicationIntervalSecs); // THEN var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); @@ -144,7 +144,7 @@ class DeduplicatePersistenceStrategyTest { int deduplicationIntervalSecs = (int) Duration.ofDays(1L).toSeconds(); // max deduplication interval duration // WHEN - strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + strategy = new DeduplicateProcessingStrategy(deduplicationIntervalSecs); // THEN var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); @@ -160,7 +160,7 @@ class DeduplicatePersistenceStrategyTest { long ts = 1_000_000L; UUID originator = UUID.randomUUID(); - assertThat(strategy.shouldPersist(ts, originator)).isTrue(); + assertThat(strategy.shouldProcess(ts, originator)).isTrue(); } @Test @@ -169,11 +169,11 @@ class DeduplicatePersistenceStrategyTest { UUID originator = UUID.randomUUID(); // Initial call should return true - assertThat(strategy.shouldPersist(baseTs, originator)).isTrue(); + assertThat(strategy.shouldProcess(baseTs, originator)).isTrue(); // Subsequent call within the same interval should return false for the same originator long withinSameIntervalTs = baseTs + 1000L; - assertThat(strategy.shouldPersist(withinSameIntervalTs, originator)).isFalse(); + assertThat(strategy.shouldProcess(withinSameIntervalTs, originator)).isFalse(); } @Test @@ -183,12 +183,12 @@ class DeduplicatePersistenceStrategyTest { UUID originator2 = UUID.randomUUID(); // First call for different originators in the same interval should return true independently - assertThat(strategy.shouldPersist(baseTs, originator1)).isTrue(); - assertThat(strategy.shouldPersist(baseTs, originator2)).isTrue(); + assertThat(strategy.shouldProcess(baseTs, originator1)).isTrue(); + assertThat(strategy.shouldProcess(baseTs, originator2)).isTrue(); // Subsequent calls for the same originators within the same interval should return false - assertThat(strategy.shouldPersist(baseTs + 500L, originator1)).isFalse(); - assertThat(strategy.shouldPersist(baseTs + 500L, originator2)).isFalse(); + assertThat(strategy.shouldProcess(baseTs + 500L, originator1)).isFalse(); + assertThat(strategy.shouldProcess(baseTs + 500L, originator2)).isFalse(); } @Test @@ -197,11 +197,11 @@ class DeduplicatePersistenceStrategyTest { long maxTs = Long.MAX_VALUE; UUID originator = UUID.randomUUID(); - assertThat(strategy.shouldPersist(minTs, originator)).isTrue(); - assertThat(strategy.shouldPersist(minTs + 1L, originator)).isFalse(); + assertThat(strategy.shouldProcess(minTs, originator)).isTrue(); + assertThat(strategy.shouldProcess(minTs + 1L, originator)).isFalse(); - assertThat(strategy.shouldPersist(maxTs, originator)).isTrue(); - assertThat(strategy.shouldPersist(maxTs - 1L, originator)).isFalse(); + assertThat(strategy.shouldProcess(maxTs, originator)).isTrue(); + assertThat(strategy.shouldProcess(maxTs - 1L, originator)).isFalse(); } @Test @@ -213,22 +213,22 @@ class DeduplicatePersistenceStrategyTest { long firstIntervalEnd = firstIntervalStart + Duration.ofSeconds(deduplicationIntervalSecs).toMillis() - 1L; long firstIntervalMiddle = calculateMiddle(firstIntervalStart, firstIntervalEnd); - assertThat(strategy.shouldPersist(firstIntervalStart, originator)).isTrue(); - assertThat(strategy.shouldPersist(firstIntervalStart + 1, originator)).isFalse(); - assertThat(strategy.shouldPersist(firstIntervalMiddle, originator)).isFalse(); - assertThat(strategy.shouldPersist(firstIntervalEnd - 1, originator)).isFalse(); - assertThat(strategy.shouldPersist(firstIntervalEnd, originator)).isFalse(); + assertThat(strategy.shouldProcess(firstIntervalStart, originator)).isTrue(); + assertThat(strategy.shouldProcess(firstIntervalStart + 1, originator)).isFalse(); + assertThat(strategy.shouldProcess(firstIntervalMiddle, originator)).isFalse(); + assertThat(strategy.shouldProcess(firstIntervalEnd - 1, originator)).isFalse(); + assertThat(strategy.shouldProcess(firstIntervalEnd, originator)).isFalse(); // check 2nd interval long secondIntervalStart = firstIntervalEnd + 1L; long secondIntervalEnd = secondIntervalStart + Duration.ofSeconds(deduplicationIntervalSecs).toMillis() - 1L; long secondIntervalMiddle = calculateMiddle(secondIntervalStart, secondIntervalEnd); - assertThat(strategy.shouldPersist(secondIntervalStart, originator)).isTrue(); - assertThat(strategy.shouldPersist(secondIntervalStart + 1, originator)).isFalse(); - assertThat(strategy.shouldPersist(secondIntervalMiddle, originator)).isFalse(); - assertThat(strategy.shouldPersist(secondIntervalEnd - 1, originator)).isFalse(); - assertThat(strategy.shouldPersist(secondIntervalEnd, originator)).isFalse(); + assertThat(strategy.shouldProcess(secondIntervalStart, originator)).isTrue(); + assertThat(strategy.shouldProcess(secondIntervalStart + 1, originator)).isFalse(); + assertThat(strategy.shouldProcess(secondIntervalMiddle, originator)).isFalse(); + assertThat(strategy.shouldProcess(secondIntervalEnd - 1, originator)).isFalse(); + assertThat(strategy.shouldProcess(secondIntervalEnd, originator)).isFalse(); } @Test @@ -238,19 +238,19 @@ class DeduplicatePersistenceStrategyTest { long baseTs = 0L; // First interval for both originators - assertThat(strategy.shouldPersist(baseTs, originator1)).isTrue(); - assertThat(strategy.shouldPersist(baseTs, originator2)).isTrue(); + assertThat(strategy.shouldProcess(baseTs, originator1)).isTrue(); + assertThat(strategy.shouldProcess(baseTs, originator2)).isTrue(); // Move to the next interval long nextIntervalTs = baseTs + Duration.ofSeconds(10).toMillis(); // Each originator should be allowed again in the new interval - assertThat(strategy.shouldPersist(nextIntervalTs, originator1)).isTrue(); - assertThat(strategy.shouldPersist(nextIntervalTs, originator2)).isTrue(); + assertThat(strategy.shouldProcess(nextIntervalTs, originator1)).isTrue(); + assertThat(strategy.shouldProcess(nextIntervalTs, originator2)).isTrue(); // Subsequent calls in the same new interval should return false - assertThat(strategy.shouldPersist(nextIntervalTs + 500L, originator1)).isFalse(); - assertThat(strategy.shouldPersist(nextIntervalTs + 500L, originator2)).isFalse(); + assertThat(strategy.shouldProcess(nextIntervalTs + 500L, originator1)).isFalse(); + assertThat(strategy.shouldProcess(nextIntervalTs + 500L, originator2)).isFalse(); } private static long calculateMiddle(long start, long end) { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessageProcessingStrategyTest.java similarity index 89% rename from rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java rename to rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessageProcessingStrategyTest.java index 125da3a495..36858418c3 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessageProcessingStrategyTest.java @@ -24,13 +24,13 @@ import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; -class OnEveryMessagePersistenceStrategyTest { +class OnEveryMessageProcessingStrategyTest { @ParameterizedTest @MethodSource("edgeCaseProvider") void shouldAlwaysReturnTrueForAnyInput(long timestamp, UUID originator) { - var onEveryMessage = OnEveryMessagePersistenceStrategy.getInstance(); - assertThat(onEveryMessage.shouldPersist(timestamp, originator)).isTrue(); + var onEveryMessage = OnEveryMessageProcessingStrategy.getInstance(); + assertThat(onEveryMessage.shouldProcess(timestamp, originator)).isTrue(); } private static Stream edgeCaseProvider() { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/ProcessingStrategyTest.java similarity index 76% rename from rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java rename to rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/ProcessingStrategyTest.java index 71a8eabed5..45fed469d2 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/ProcessingStrategyTest.java @@ -22,23 +22,23 @@ import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; -class PersistenceStrategyTest { +class ProcessingStrategyTest { @Test void testOnEveryMessageReturnsCorrectInstance() { - PersistenceStrategy strategy = PersistenceStrategy.onEveryMessage(); + ProcessingStrategy strategy = ProcessingStrategy.onEveryMessage(); assertThat(strategy) .isNotNull() - .isInstanceOf(OnEveryMessagePersistenceStrategy.class); + .isInstanceOf(OnEveryMessageProcessingStrategy.class); } @Test void testDeduplicateReturnsCorrectInstance() { int validDeduplicationIntervalSecs = 5; - PersistenceStrategy strategy = PersistenceStrategy.deduplicate(validDeduplicationIntervalSecs); + ProcessingStrategy strategy = ProcessingStrategy.deduplicate(validDeduplicationIntervalSecs); assertThat(strategy) .isNotNull() - .isInstanceOf(DeduplicatePersistenceStrategy.class); + .isInstanceOf(DeduplicateProcessingStrategy.class); long actualDeduplicationIntervalMillis = (long) ReflectionTestUtils.getField(strategy, "deduplicationIntervalMillis"); assertThat(actualDeduplicationIntervalMillis).isEqualTo(Duration.ofSeconds(validDeduplicationIntervalSecs).toMillis()); @@ -46,10 +46,10 @@ class PersistenceStrategyTest { @Test void testSkipReturnsCorrectInstance() { - PersistenceStrategy strategy = PersistenceStrategy.skip(); + ProcessingStrategy strategy = ProcessingStrategy.skip(); assertThat(strategy) .isNotNull() - .isInstanceOf(SkipPersistenceStrategy.class); + .isInstanceOf(SkipProcessingStrategy.class); } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipProcessingStrategyTest.java similarity index 90% rename from rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java rename to rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipProcessingStrategyTest.java index 1a63ce7460..ffdf2fc87e 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipProcessingStrategyTest.java @@ -24,13 +24,13 @@ import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; -class SkipPersistenceStrategyTest { +class SkipProcessingStrategyTest { @ParameterizedTest @MethodSource("edgeCaseProvider") void shouldAlwaysReturnFalseForAnyInput(long timestamp, UUID originator) { - var skipStrategy = SkipPersistenceStrategy.getInstance(); - assertThat(skipStrategy.shouldPersist(timestamp, originator)).isFalse(); + var skipStrategy = SkipProcessingStrategy.getInstance(); + assertThat(skipStrategy.shouldProcess(timestamp, originator)).isFalse(); } private static Stream edgeCaseProvider() { From fc3bc522470ae75c9c9f6a926f17257b86b4bd1f Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 7 Feb 2025 12:15:32 +0200 Subject: [PATCH 215/416] UI: [Rule node save ts] Rename Persistence settings to Processing settings --- ...anced-persistence-setting-row.component.ts | 22 +++--- ...dvanced-persistence-setting.component.html | 5 ++ .../advanced-persistence-setting.component.ts | 5 +- .../action/timeseries-config.component.html | 13 ++-- .../action/timeseries-config.component.ts | 78 +++++++++---------- .../action/timeseries-config.models.ts | 54 ++++++------- .../common/example-hint.component.html | 6 +- .../common/example-hint.component.scss | 31 -------- .../common/example-hint.component.ts | 2 +- .../rulenode/save_timeseries_node_advanced.md | 3 + .../assets/locale/locale.constant-en_US.json | 5 +- 11 files changed, 100 insertions(+), 124 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/components/rule-node/common/example-hint.component.scss create mode 100644 ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts index 64ea045f8f..bf9b48aca3 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts @@ -24,11 +24,11 @@ import { Validator } from '@angular/forms'; import { - AdvancedPersistenceConfig, - defaultAdvancedPersistenceConfig, + AdvancedProcessingConfig, + defaultAdvancedProcessingConfig, maxDeduplicateTimeSecs, - PersistenceType, - PersistenceTypeTranslationMap + ProcessingType, + ProcessingTypeTranslationMap } from '@home/components/rule-node/action/timeseries-config.models'; import { isDefinedAndNotNull } from '@core/utils'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @@ -52,13 +52,13 @@ export class AdvancedPersistenceSettingRowComponent implements ControlValueAcces title: string; persistenceSettingRowForm = this.fb.group({ - type: [defaultAdvancedPersistenceConfig.type], + type: [defaultAdvancedProcessingConfig.type], deduplicationIntervalSecs: [{value: 60, disabled: true}] }); - PersistenceType = PersistenceType; - persistenceStrategies = [PersistenceType.ON_EVERY_MESSAGE, PersistenceType.DEDUPLICATE, PersistenceType.SKIP]; - PersistenceTypeTranslationMap = PersistenceTypeTranslationMap; + PersistenceType = ProcessingType; + persistenceStrategies = [ProcessingType.ON_EVERY_MESSAGE, ProcessingType.DEDUPLICATE, ProcessingType.SKIP]; + PersistenceTypeTranslationMap = ProcessingTypeTranslationMap; maxDeduplicateTime = maxDeduplicateTimeSecs; @@ -96,16 +96,16 @@ export class AdvancedPersistenceSettingRowComponent implements ControlValueAcces }; } - writeValue(value: AdvancedPersistenceConfig) { + writeValue(value: AdvancedProcessingConfig) { if (isDefinedAndNotNull(value)) { this.persistenceSettingRowForm.patchValue(value, {emitEvent: false}); } else { - this.persistenceSettingRowForm.patchValue(defaultAdvancedPersistenceConfig); + this.persistenceSettingRowForm.patchValue(defaultAdvancedProcessingConfig); } } private updatedValidation() { - if (this.persistenceSettingRowForm.get('type').value === PersistenceType.DEDUPLICATE) { + if (this.persistenceSettingRowForm.get('type').value === ProcessingType.DEDUPLICATE) { this.persistenceSettingRowForm.get('deduplicationIntervalSecs').enable({emitEvent: false}); } else { this.persistenceSettingRowForm.get('deduplicationIntervalSecs').disable({emitEvent: false}) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html index eb0f05bec2..d3cbb1cdcb 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html @@ -16,6 +16,11 @@ -->
        + +
        -
        +
        -
        - rule-node-config.save-time-series.persistence-settings +
        + rule-node-config.save-time-series.processing-settings
        @@ -27,7 +27,7 @@ {{ 'rule-node-config.advanced-mode' | translate }}
        - @if(!timeseriesConfigForm.get('persistenceSettings.isAdvanced').value) { + @if(!timeseriesConfigForm.get('processingSettings.isAdvanced').value) { rule-node-config.save-time-series.strategy @@ -37,7 +37,7 @@ - @if(timeseriesConfigForm.get('persistenceSettings.type').value === PersistenceType.DEDUPLICATE) { + @if(timeseriesConfigForm.get('processingSettings.type').value === PersistenceType.DEDUPLICATE) { } - } - @else { + } @else { { - persistenceSettings: PersistenceSettingsForm +export interface TimeseriesNodeConfigurationForm extends Omit { + processingSettings: ProcessingSettingsForm } -export type PersistenceSettings = BasicPersistenceSettings & Partial & Partial; +export type ProcessingSettings = ProcessingSettingsSettings & Partial & Partial; -export type PersistenceSettingsForm = Omit & { +export type ProcessingSettingsForm = Omit & { isAdvanced: boolean; - advanced?: Partial; - type: PersistenceType; + advanced?: Partial; + type: ProcessingType; }; -export enum PersistenceType { +export enum ProcessingType { ON_EVERY_MESSAGE = 'ON_EVERY_MESSAGE', DEDUPLICATE = 'DEDUPLICATE', WEBSOCKETS_ONLY = 'WEBSOCKETS_ONLY', @@ -44,35 +44,35 @@ export enum PersistenceType { SKIP = 'SKIP' } -export const PersistenceTypeTranslationMap = new Map([ - [PersistenceType.ON_EVERY_MESSAGE, 'rule-node-config.save-time-series.strategy-type.every-message'], - [PersistenceType.DEDUPLICATE, 'rule-node-config.save-time-series.strategy-type.deduplicate'], - [PersistenceType.WEBSOCKETS_ONLY, 'rule-node-config.save-time-series.strategy-type.web-sockets-only'], - [PersistenceType.SKIP, 'rule-node-config.save-time-series.strategy-type.skip'], +export const ProcessingTypeTranslationMap = new Map([ + [ProcessingType.ON_EVERY_MESSAGE, 'rule-node-config.save-time-series.strategy-type.every-message'], + [ProcessingType.DEDUPLICATE, 'rule-node-config.save-time-series.strategy-type.deduplicate'], + [ProcessingType.WEBSOCKETS_ONLY, 'rule-node-config.save-time-series.strategy-type.web-sockets-only'], + [ProcessingType.SKIP, 'rule-node-config.save-time-series.strategy-type.skip'], ]) -export interface BasicPersistenceSettings { - type: PersistenceType; +export interface ProcessingSettingsSettings { + type: ProcessingType; } -export interface DeduplicatePersistenceStrategy extends BasicPersistenceSettings{ +export interface DeduplicateProcessingStrategy extends ProcessingSettingsSettings{ deduplicationIntervalSecs: number; } -export interface AdvancedPersistenceStrategy extends BasicPersistenceSettings{ - timeseries: AdvancedPersistenceConfig; - latest: AdvancedPersistenceConfig; - webSockets: AdvancedPersistenceConfig; +export interface AdvancedProcessingStrategy extends ProcessingSettingsSettings{ + timeseries: AdvancedProcessingConfig; + latest: AdvancedProcessingConfig; + webSockets: AdvancedProcessingConfig; } -export type AdvancedPersistenceConfig = WithOptional; +export type AdvancedProcessingConfig = WithOptional; -export const defaultAdvancedPersistenceConfig: AdvancedPersistenceConfig = { - type: PersistenceType.ON_EVERY_MESSAGE +export const defaultAdvancedProcessingConfig: AdvancedProcessingConfig = { + type: ProcessingType.ON_EVERY_MESSAGE } -export const defaultAdvancedPersistenceStrategy: Omit = { - timeseries: defaultAdvancedPersistenceConfig, - latest: defaultAdvancedPersistenceConfig, - webSockets: defaultAdvancedPersistenceConfig, +export const defaultAdvancedPersistenceStrategy: Omit = { + timeseries: defaultAdvancedProcessingConfig, + latest: defaultAdvancedProcessingConfig, + webSockets: defaultAdvancedProcessingConfig, } diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/example-hint.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/example-hint.component.html index 238fdce6af..0981daa5cb 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/example-hint.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/example-hint.component.html @@ -15,11 +15,11 @@ limitations under the License. --> -
        +
        -
        +
        Date: Fri, 7 Feb 2025 12:19:37 +0200 Subject: [PATCH 216/416] UI: [Rule node save ts] Fixed typo --- .../rule-node/action/timeseries-config.models.ts | 8 ++++---- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts index dfee001181..e9d024663d 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts @@ -28,7 +28,7 @@ export interface TimeseriesNodeConfigurationForm extends Omit & Partial; +export type ProcessingSettings = BasicProcessingSettings & Partial & Partial; export type ProcessingSettingsForm = Omit & { isAdvanced: boolean; @@ -51,15 +51,15 @@ export const ProcessingTypeTranslationMap = new Map([ [ProcessingType.SKIP, 'rule-node-config.save-time-series.strategy-type.skip'], ]) -export interface ProcessingSettingsSettings { +export interface BasicProcessingSettings { type: ProcessingType; } -export interface DeduplicateProcessingStrategy extends ProcessingSettingsSettings{ +export interface DeduplicateProcessingStrategy extends BasicProcessingSettings{ deduplicationIntervalSecs: number; } -export interface AdvancedProcessingStrategy extends ProcessingSettingsSettings{ +export interface AdvancedProcessingStrategy extends BasicProcessingSettings{ timeseries: AdvancedProcessingConfig; latest: AdvancedProcessingConfig; webSockets: AdvancedProcessingConfig; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 9ca1e2dd4d..d3f6e3b2fc 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5136,7 +5136,7 @@ "save-time-series": { "processing-settings": "Processing settings", "processing-settings-hint": "Define how time series data is processed. In Basic mode, select a preconfigured processing strategy or enable only WebSocket updates. Advanced mode allows you to select individual processing strategies for each action.", - "advanced-settings-hint": "Advanced settings mode is dangerous, and if you do not understand it, you may lose telemetry.\n\n\n\n\n\n\n\n", + "advanced-settings-hint": "Advanced settings mode is dangerous, and if you do not understand it, you may lose telemetry.", "strategy": "Strategy", "deduplication-interval": "Deduplication interval", "deduplication-interval-required": "Deduplication interval is required", From 5d47ef230918167fed530cb7d82d39e9abd6622d Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 7 Feb 2025 13:32:59 +0200 Subject: [PATCH 217/416] UI: Fixed a blur effect issue in the Getting started widget --- .../getting-started-widget.component.html | 26 +++++++++---------- .../getting-started-widget.component.scss | 3 +++ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/home-page/getting-started-widget.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/home-page/getting-started-widget.component.html index 004d9807b7..c0f980ca2e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/home-page/getting-started-widget.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/home-page/getting-started-widget.component.html @@ -26,7 +26,7 @@ -
        +
        widgets.getting-started.sys-admin.step1.title
        {{ 'tenant.tenants' | translate }}
        @@ -37,7 +37,7 @@ -
        +
        widgets.getting-started.sys-admin.step2.title
        {{ 'admin.settings' | translate }}
        @@ -48,7 +48,7 @@ -
        +
        widgets.getting-started.sys-admin.step3.title
        {{ 'admin.settings' | translate }}
        @@ -59,7 +59,7 @@ -
        +
        widgets.getting-started.sys-admin.step4.title
        {{ 'admin.settings' | translate }}
        @@ -70,7 +70,7 @@ -
        +
        widgets.getting-started.sys-admin.step5.title
        {{ 'admin.settings' | translate }}
        @@ -81,7 +81,7 @@ -
        +
        widgets.getting-started.sys-admin.step6.title
        {{ 'admin.settings' | translate }}
        @@ -94,7 +94,7 @@ -
        +
        widgets.getting-started.tenant-admin.step1.title
        {{ 'device.devices' | translate }}
        @@ -105,7 +105,7 @@ -
        +
        widgets.getting-started.tenant-admin.step2.title
        @@ -153,7 +153,7 @@ -
        +
        widgets.getting-started.tenant-admin.step3.title
        {{ 'dashboard.dashboards' | translate }}
        @@ -164,7 +164,7 @@ -
        +
        widgets.getting-started.tenant-admin.step4.title
        {{ 'widgets.getting-started.tenant-admin.step4.alarm-rules' | translate }}
        @@ -175,7 +175,7 @@ -
        +
        widgets.getting-started.tenant-admin.step5.title
        @@ -193,7 +193,7 @@ -
        +
        widgets.getting-started.tenant-admin.step6.title
        {{ 'customer.customers' | translate }}
        @@ -205,7 +205,7 @@ -
        +
        diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/home-page/getting-started-widget.component.scss b/ui-ngx/src/app/modules/home/components/widget/lib/home-page/getting-started-widget.component.scss index e684c094d7..4a21006cc0 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/home-page/getting-started-widget.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/lib/home-page/getting-started-widget.component.scss @@ -38,6 +38,9 @@ .tb-get-started-container { flex: 1; overflow: auto; + mat-stepper { + transform: scale(1); //fixed blur content + } } } From c27c82b80d5943b169b6bbc37715cb1d0fe8790e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 14:01:02 +0200 Subject: [PATCH 218/416] updated test expression endpoint --- .../server/controller/CalculatedFieldController.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 435e46069d..5c8b40dcad 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -48,7 +48,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngi import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -85,7 +85,6 @@ public class CalculatedFieldController extends BaseController { + MARKDOWN_CODE_BLOCK_START + "{\n" + " \"expression\": \"var temp = 0; foreach(element: temperature.entrySet()) { temp += element.getValue(); } var avgTemperature = temp / temperature.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity; return { \\\"adjustedTemperature\\\": adjustedTemperature };\",\n" + - " \"argNames\": [\"temperature\", \"humidity\"],\n" + " \"arguments\": {\n" + " \"temperature\": {\n" + " \"14327856345\": 22.4,\n" + @@ -173,12 +172,12 @@ public class CalculatedFieldController extends BaseController { @io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") @RequestBody JsonNode inputParams) { String expression = inputParams.get("expression").asText(); - String[] argNames = JacksonUtil.treeToValue(inputParams.get("argNames"), String[].class); Map arguments = Objects.requireNonNullElse( JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference>() { }), Collections.emptyMap() ); + ArrayList argNames = new ArrayList<>(arguments.keySet()); String output = ""; String errorText = ""; @@ -192,10 +191,10 @@ public class CalculatedFieldController extends BaseController { getTenantId(), tbelInvokeService, expression, - argNames + argNames.toArray(String[]::new) ); - Object[] args = Arrays.stream(argNames) + Object[] args = argNames.stream() .map(arguments::get) .toArray(); From fe4d2ba49759f8dfcf9313e4777bce69235c7c7f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 14:24:39 +0200 Subject: [PATCH 219/416] added tests --- ...alculatedFieldManagerMessageProcessor.java | 10 +- .../cf/CalculatedFieldIntegrationTest.java | 266 ++++++++++++++++++ .../CalculatedFieldConfiguration.java | 2 + 3 files changed, 269 insertions(+), 9 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 7b03ed2938..6a6edae768 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -258,14 +258,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); - try { - newCfCtx.init(); - } catch (Exception e) { - log.debug("[{}] Failed to initialize CF context.", newCf.getId(), e); - if (DebugModeUtil.isDebugAllAvailable(newCf)) { - systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); - } - } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); @@ -289,7 +281,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); - if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { + if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { try { newCfCtx.init(); } catch (Exception e) { diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java new file mode 100644 index 0000000000..fda0724b1b --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -0,0 +1,266 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.controller.CalculatedFieldControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +@DaoSqlTest +public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { + + @BeforeEach + void setUp() throws Exception { + loginTenantAdmin(); + } + + @Test + public void testSimpleCalculatedField() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + JsonNode timeSeries = JacksonUtil.toJsonNode("{\"temperature\":25}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, timeSeries); + + JsonNode attributes = JacksonUtil.toJsonNode("{\"deviceTemperature\":40}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> perform initial calculation + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + + // update telemetry -> recalculate state + JsonNode newTelemetry = JacksonUtil.toJsonNode("{\"temperature\":30}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, newTelemetry); + + Thread.sleep(300); + + ObjectNode fahrenheitTempAfterUpdate = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); + assertThat(fahrenheitTempAfterUpdate).isNotNull(); + assertThat(fahrenheitTempAfterUpdate.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + + // update CF output -> perform calculation with updated output + Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); + savedOutput.setType(OutputType.ATTRIBUTES); + savedOutput.setScope(AttributeScope.SERVER_SCOPE); + savedOutput.setName("temperatureF"); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + Thread.sleep(300); + + ArrayNode temperatureF = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + + // update CF argument -> perform calculation with new argument + + Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); + savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + Thread.sleep(300); + + ArrayNode temperatureFAfterUpdateArg = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + assertThat(temperatureFAfterUpdateArg).isNotNull(); + assertThat(temperatureFAfterUpdateArg.get(0).get("value").asText()).isEqualTo("104.0"); + + // update CF expression -> perform calculation with new expression + savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + Thread.sleep(300); + + ArrayNode temperatureFAfterUpdateExpression = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + assertThat(temperatureFAfterUpdateExpression).isNotNull(); + assertThat(temperatureFAfterUpdateExpression.get(0).get("value").asText()).isEqualTo("104.0"); + } + + @Test + public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + JsonNode deviceAttributes = JacksonUtil.toJsonNode("{\"x\":40}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, deviceAttributes); + + AssetProfile assetProfile = doPost("/api/assetProfile", createAssetProfile("Test Asset Profile"), AssetProfile.class); + + Asset asset1 = new Asset(); + asset1.setName("Test asset 1"); + asset1.setAssetProfileId(assetProfile.getId()); + + Asset savedAsset1 = doPost("/api/asset", asset1, Asset.class); + + JsonNode asset1Attributes = JacksonUtil.toJsonNode("{\"y\":11}"); + doPost("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset1Attributes); + + Asset asset2 = new Asset(); + asset2.setName("Test asset 2"); + asset2.setAssetProfileId(assetProfile.getId()); + + Asset savedAsset2 = doPost("/api/asset", asset2, Asset.class); + + JsonNode asset2Attributes = JacksonUtil.toJsonNode("{\"y\":12}"); + doPost("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset2Attributes); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(assetProfile.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("z = x + y"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("y", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument1.setRefEntityKey(refEntityKey1); + + Argument argument2 = new Argument(); + argument2.setRefEntityId(testDevice.getId()); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("x", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument2.setRefEntityKey(refEntityKey2); + + config.setArguments(Map.of("x", argument2, "y", argument1)); + + config.setExpression("x + y"); + + Output output = new Output(); + output.setName("z"); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF and perform initial calculation + doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + // result of asset 1 + ArrayNode z1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); + + // result of asset 2 + ArrayNode z2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + + // update device telemetry -> recalculate state for all assets + JsonNode updatedDeviceAttributes = JacksonUtil.toJsonNode("{\"x\":25}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedDeviceAttributes); + + Thread.sleep(300); + + // result of asset 1 + ArrayNode updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(updZ1).isNotNull(); + assertThat(updZ1.get(0).get("value").asText()).isEqualTo("36.0"); + + // result of asset 2 + ArrayNode updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(updZ2).isNotNull(); + assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); + +// // update asset 1 telemetry -> recalculate state only for asset 1 +// JsonNode updatedAsset1Attributes = JacksonUtil.toJsonNode("{\"x\":15}"); +// doPost("/api/plugins/telemetry/DEVICE/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset1Attributes); +// +// Thread.sleep(300); +// +// // result of asset 1 +// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ1).isNotNull(); +// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); +// +// // result of asset 2 (no changes) +// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ2).isNotNull(); +// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); +// +// // update asset 2 telemetry -> recalculate state only for asset 2 +// JsonNode updatedAsset2Attributes = JacksonUtil.toJsonNode("{\"x\":5}"); +// doPost("/api/plugins/telemetry/DEVICE/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset2Attributes); +// +// Thread.sleep(300); +// +// // result of asset 1 (no changes) +// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ1).isNotNull(); +// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); +// +// // result of asset 2 +// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ2).isNotNull(); +// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("30.0"); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 8f56bf491d..9bf3d728aa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -46,6 +46,8 @@ public interface CalculatedFieldConfiguration { String getExpression(); + void setExpression(String expression); + Output getOutput(); @JsonIgnore From 96e292fac5cb76511b645553ed83fe7b6b4b53d7 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 7 Feb 2025 15:31:21 +0200 Subject: [PATCH 220/416] Calculated field fixes and improvements --- ...lated-field-arguments-table.component.html | 7 ++-- ...culated-field-arguments-table.component.ts | 20 ++++------ .../calculated-field-dialog.component.html | 2 +- ...ulated-field-argument-panel.component.html | 37 +++++++++++-------- ...ulated-field-argument-panel.component.scss | 22 +++++++++++ ...lculated-field-argument-panel.component.ts | 24 +++++++++++- .../import-export/import-export.service.ts | 1 - .../assets/locale/locale.constant-en_US.json | 1 + 8 files changed, 78 insertions(+), 36 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 4b7e516db0..d8a6c7cdb0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -88,9 +88,8 @@ [matTooltip]="'action.edit' | translate" matTooltipPosition="above"> {{ 'calculated-fields.no-arguments' | translate }} }
        - @if (errorText && this.argumentsFormArray.dirty) { + @if (errorText) { }
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index a004e3db6f..171c382b7a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -17,9 +17,7 @@ import { ChangeDetectorRef, Component, - effect, forwardRef, - input, Input, OnChanges, Renderer2, @@ -77,8 +75,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces @Input() entityId: EntityId; @Input() tenantId: string; @Input() entityName: string; - - calculatedFieldType = input() + @Input() calculatedFieldType: CalculatedFieldType; errorText = ''; argumentsFormArray = this.fb.array([]); @@ -103,17 +100,12 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { this.propagateChange(this.getArgumentsObject()); }); - effect(() => { - if (this.calculatedFieldType() && this.argumentsFormArray.dirty) { - this.argumentsFormArray.updateValueAndValidity(); - } - }); } ngOnChanges(changes: SimpleChanges): void { if (changes.calculatedFieldType?.previousValue && changes.calculatedFieldType.currentValue !== changes.calculatedFieldType.previousValue) { - this.argumentsFormArray.markAsDirty(); + this.argumentsFormArray.updateValueAndValidity(); } } @@ -142,14 +134,16 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces if (this.popoverService.hasPopover(trigger)) { this.popoverService.hidePopover(trigger); } else { + const argumentObj = this.argumentsFormArray.at(index)?.getRawValue() ?? {}; const ctx = { index, - argument: this.argumentsFormArray.at(index)?.getRawValue() ?? {}, + argument: argumentObj, entityId: this.entityId, - calculatedFieldType: this.calculatedFieldType(), + calculatedFieldType: this.calculatedFieldType, buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, entityName: this.entityName, + argumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argumentObj.argumentName), }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, @@ -171,7 +165,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } private updateErrorText(): void { - if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE + if (this.calculatedFieldType === CalculatedFieldType.SIMPLE && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; } else if (!this.argumentsFormArray.controls.length) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index ac60ae8178..1d708f060a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -65,7 +65,7 @@
        -
        {{ 'calculated-fields.arguments' | translate }}
        +
        {{ 'calculated-fields.arguments' | translate }}*
        warning - } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) { + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('duplicateName')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) {
        } @else { -
        -
        {{ 'calculated-fields.attribute-scope' | translate }}
        - - - - {{ 'calculated-fields.server-attributes' | translate }} - - @if (entityType === ArgumentEntityType.Device - || entityType === ArgumentEntityType.Current && entityId.entityType === EntityType.DEVICE) { + @if (isDeviceEntity) { +
        +
        {{ 'calculated-fields.attribute-scope' | translate }}
        + + + + {{ 'calculated-fields.server-attributes' | translate }} + {{ 'calculated-fields.client-attributes' | translate }} {{ 'calculated-fields.shared-attributes' | translate }} - } - - -
        +         +         +
        + }
        {{ 'calculated-fields.attribute-key' | translate }}
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..45c17628d5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + .time-window-field { + .mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field .mdc-notched-outline__notch { + border-left: 1px solid rgba(0, 0, 0, 0) !important; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 510bdd95f3..1632bd9311 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -16,7 +16,7 @@ import { ChangeDetectorRef, Component, Input, OnInit, output } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; -import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { FormBuilder, FormGroup, UntypedFormControl, ValidatorFn, Validators } from '@angular/forms'; import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, @@ -42,6 +42,7 @@ import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] }) export class CalculatedFieldArgumentPanelComponent implements OnInit { @@ -52,11 +53,12 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { @Input() tenantId: string; @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; + @Input() argumentNames: string[]; argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, this.uniqNameRequired(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -109,6 +111,12 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { return this.argumentFormGroup.get('refEntityKey') as FormGroup; } + get isDeviceEntity(): boolean { + return this.entityType === ArgumentEntityType.Device + || (this.entityType === ArgumentEntityType.Current + && (this.entityId.entityType === EntityType.DEVICE || this.entityId.entityType === EntityType.DEVICE_PROFILE)) + } + ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); @@ -188,9 +196,21 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { this.argumentFormGroup.get('refEntityId').get('id').setValue(''); this.argumentFormGroup.get('refEntityId') .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); + if (!this.isDeviceEntity) { + this.refEntityKeyFormGroup.get('scope').setValue(AttributeScope.SERVER_SCOPE); + } }); } + private uniqNameRequired(): ValidatorFn { + return (control: UntypedFormControl) => { + const newName = control.value.trim().toLowerCase(); + const isDuplicate = this.argumentNames?.some(name => name.toLowerCase() === newName); + + return isDuplicate ? { duplicateName: true } : null; + }; + } + private observeEntityKeyChanges(): void { this.argumentFormGroup.get('refEntityKey').get('type').valueChanges .pipe(takeUntilDestroyed()) diff --git a/ui-ngx/src/app/shared/import-export/import-export.service.ts b/ui-ngx/src/app/shared/import-export/import-export.service.ts index 0a49acf8b6..18000bcd4f 100644 --- a/ui-ngx/src/app/shared/import-export/import-export.service.ts +++ b/ui-ngx/src/app/shared/import-export/import-export.service.ts @@ -997,7 +997,6 @@ export class ImportExportService { && !!Object.keys(configuration.arguments).length && isDefined(configuration.expression) && isDefined(configuration.output) - && isNotEmptyStr(configuration.output.name); } private validateImportedImage(image: ImageExportData): boolean { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 9bdf533fef..d17409e52d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1064,6 +1064,7 @@ "expression-max-length": "Expression length should be less than 255 characters.", "argument-name-required": "Argument name is required.", "argument-name-pattern": "Argument name is invalid.", + "argument-name-duplicate": "Argument with such name already exists.", "argument-name-max-length": "Argument name should be less than 256 characters.", "argument-type-required": "Argument type is required." } From 8e3c4dc18c3de285f64b111fe62affb03710416a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 7 Feb 2025 15:46:25 +0200 Subject: [PATCH 221/416] Cluster mode fixes --- .../server/actors/ActorSystemContext.java | 6 ++ ...CalculatedFieldEntityMessageProcessor.java | 26 ++++--- ...alculatedFieldManagerMessageProcessor.java | 5 +- .../cf/CalculatedFieldExecutionService.java | 3 - ...efaultCalculatedFieldExecutionService.java | 11 --- .../cf/DefaultCalculatedFieldInitService.java | 9 --- .../server/service/cf/RocksDBService.java | 3 +- ...aultCalculatedFieldEntityProfileCache.java | 1 + .../cf/ctx/CalculatedFieldStateService.java | 4 -- .../KafkaCalculatedFieldStateService.java | 68 +++++++++++++++++++ ...> RocksDBCalculatedFieldStateService.java} | 17 +++-- .../queue/DefaultTbClusterService.java | 10 +-- 12 files changed, 116 insertions(+), 47 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{RocksDBStateService.java => RocksDBCalculatedFieldStateService.java} (92%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 8f15a6766b..c714ed7dab 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -108,6 +108,7 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -527,6 +528,11 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldStateService calculatedFieldStateService; + @Lazy @Autowired(required = false) @Getter diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 544dea44c9..0d327adca9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

        * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

        + * http://www.apache.org/licenses/LICENSE-2.0 + *

        * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,6 +38,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -67,6 +68,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM final TenantId tenantId; final EntityId entityId; final CalculatedFieldExecutionService cfService; + final CalculatedFieldStateService cfStateService; TbActorCtx ctx; Map states = new HashMap<>(); @@ -76,6 +78,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM this.tenantId = tenantId; this.entityId = entityId; this.cfService = systemContext.getCalculatedFieldExecutionService(); + this.cfStateService = systemContext.getCalculatedFieldStateService(); } void init(TbActorCtx ctx) { @@ -102,13 +105,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); - states.forEach((cfId, state) -> cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); + states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); ctx.stop(ctx.getSelf()); } else { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var state = states.remove(cfId); if (state != null) { - cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); } } } @@ -178,8 +181,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); callback.onSuccess(CALLBACKS_PER_CF); } - CalculatedFieldState state = getOrInitState(ctx); - if (state.updateState(newArgValues)) { + CalculatedFieldState state = states.get(ctx.getCfId()); + boolean justRestored = false; + if (state == null) { + state = getOrInitState(ctx); + justRestored = true; + } + if (state.updateState(newArgValues) || justRestored) { cfIdList = new ArrayList<>(cfIdList); cfIdList.add(ctx.getCfId()); processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); @@ -222,7 +230,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { callback.onSuccess(); // State was updated but no calculation performed; } - cfService.pushStateToStorage(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + cfStateService.persistState(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 6a6edae768..cdc31ed93c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -42,6 +42,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntit import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -68,6 +69,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldStateService cfStateService; private final CalculatedFieldEntityProfileCache cfEntityCache; private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; @@ -80,6 +82,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware super(systemContext); this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); this.cfExecService = systemContext.getCalculatedFieldExecutionService(); + this.cfStateService = systemContext.getCalculatedFieldStateService(); this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); @@ -127,7 +130,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); getOrCreateActor(msg.getId().entityId()).tell(msg); } else { - cfExecService.deleteStateFromStorage(msg.getId(), msg.getCallback()); + cfStateService.removeState(msg.getId(), msg.getCallback()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 393fbd3ec2..47fd560b4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -43,13 +43,10 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); - void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); - void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 43e85d87e9..e84e785a94 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -141,7 +141,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; private final ApiLimitService apiLimitService; @@ -263,16 +262,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - @Override - public void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - stateService.persistState(ctx, stateId, state, callback); - } - - @Override - public void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback) { - stateService.removeState(calculatedFieldEntityCtxId, callback); - } - @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index f71617e204..2afd6d8238 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -29,7 +29,6 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @Slf4j @Service @@ -38,9 +37,6 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { private final CalculatedFieldEntityProfileCache entityProfileCache; - private final CalculatedFieldStateService stateService; - - private final ActorSystemContext actorSystemContext; private final AssetService assetService; private final DeviceService deviceService; @@ -62,9 +58,4 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer } } - @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) - public void initCalculatedFieldStates() { - stateService.restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index fe800f61ad..7181cc43ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -21,6 +21,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.WriteOptions; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; @@ -32,7 +33,7 @@ import java.util.Map; @Service @Slf4j -@ConditionalOnExpression("'${service.type:null}'=='monolith'") +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) public class RocksDBService { private final RocksDB db; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 13f95c547d..866cc86f24 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -39,6 +39,7 @@ import java.util.stream.Collectors; @Service @Slf4j @RequiredArgsConstructor +//TODO: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { private static final Integer UNKNOWN = -1; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index e822d52767..ce1562c735 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -19,12 +19,8 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import java.util.Map; - public interface CalculatedFieldStateService { - Map restoreStates(); - void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java new file mode 100644 index 0000000000..9d45532eec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.service.cf.RocksDBService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; + +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${zk.enabled:false}'=='true' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") +public class KafkaCalculatedFieldStateService implements CalculatedFieldStateService { + + @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) + public void initCalculatedFieldStates() { + } + + @Override + public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + callback.onSuccess(); + } + + @Override + public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { + callback.onSuccess(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index b2e33e1705..86556d7adc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -16,8 +16,10 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.RequiredArgsConstructor; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -32,6 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldState import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -44,12 +47,12 @@ import java.util.stream.Collectors; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${service.type:null}'=='monolith'") -public class RocksDBStateService implements CalculatedFieldStateService { +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateService { + private final ActorSystemContext actorSystemContext; private final RocksDBService rocksDBService; - @Override public Map restoreStates() { return rocksDBService.getAll().entrySet().stream() .collect(Collectors.toMap( @@ -58,6 +61,12 @@ public class RocksDBStateService implements CalculatedFieldStateService { )); } + @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) + public void initCalculatedFieldStates() { + restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); + } + + @Override public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { CalculatedFieldStateProto stateProto = toProto(stateId, state); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index fa02435ca0..2a090fe701 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -674,7 +674,7 @@ public class DefaultTbClusterService implements TbClusterService { .oldName(old.getName()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } if (deviceNameChanged || deviceProfileChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(entity.getTenantId(), entity.getId(), entity.getName(), entity.getType()), null); @@ -687,7 +687,7 @@ public class DefaultTbClusterService implements TbClusterService { .profileId(entity.getDeviceProfileId()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(entity.getTenantId(), entity.getId(), created, !created, false); @@ -710,7 +710,7 @@ public class DefaultTbClusterService implements TbClusterService { .oldName(old.getName()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } } else { ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() @@ -720,7 +720,7 @@ public class DefaultTbClusterService implements TbClusterService { .profileId(entity.getAssetProfileId()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @@ -872,6 +872,6 @@ public class DefaultTbClusterService implements TbClusterService { private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId) { ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.DELETED); - pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } } From 148ea630c2fd8cf9d8db07c1cf06e1ce2c014e9c Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 7 Feb 2025 15:58:03 +0200 Subject: [PATCH 222/416] Save time series strategies: added description for advanced processing settings mode --- .../rulenode/save_timeseries_node_advanced.md | 30 +++++++++++++++++-- .../assets/locale/locale.constant-en_US.json | 6 ++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md b/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md index 4aae99c710..44a037f180 100644 --- a/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md +++ b/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md @@ -1,3 +1,29 @@ -**TO DO;** +#### Potential unexpected behavior with mixed processing strategies -Advanced mode doc +When configuring the processing strategies, certain combinations can lead to unexpected behavior. Consider the following scenarios: + +- **Disabling WebSocket (WS) updates** + + If WS updates are disabled, any changes to the time series data won’t be pushed to dashboards (or other WS subscriptions). + This means that even if a database is updated, dashboards may not display the updated data until browser page is reloaded. + +- **Different deduplication intervals across actions** + + When you configure different deduplication intervals for actions, the same incoming message might be processed differently for each action. + For example, a message might be stored immediately in the Time series table (if set to *On every message*) while not being stored in the Latest values table because its deduplication interval hasn’t elapsed. + Also, if the WebSocket updates are configured with a different interval, dashboards might show updates that do not match what is stored in the database. + +- **Skipping database storage** + + Choosing to disable one or more persistence actions (for instance, skipping database storage for Time series or Latest values while keeping WS updates enabled) introduces the risk of having only partial data available: + - If a message is processed only for real-time notifications (WebSockets) and not stored in the database, historical queries may not match data on the dashboard. + - When processing strategies for Time series and Latest values are out-of-sync, telemetry data may be stored in one table (e.g., Time series) while the same data is absent in the other (e.g., Latest values). + +- **Deduplication cache clearing** + + The deduplication mechanism uses a cache to track processed messages within each interval. + For performance and system stability reasons, this cache is periodically cleared. + As a result, if a cache entry is removed during the deduplication period, messages from the same originator may be processed more than once within that interval. + This means deduplication should be used as a performance optimization rather than an absolute guarantee of single processing per interval. + +We recommend using deduplication only when the occasional repeated processing is acceptable and won't cause system correctness issue or data inconsistencies. diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index d3f6e3b2fc..2ad89d67ca 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5135,8 +5135,8 @@ "advanced-mode": "Advanced", "save-time-series": { "processing-settings": "Processing settings", - "processing-settings-hint": "Define how time series data is processed. In Basic mode, select a preconfigured processing strategy or enable only WebSocket updates. Advanced mode allows you to select individual processing strategies for each action.", - "advanced-settings-hint": "Advanced settings mode is dangerous, and if you do not understand it, you may lose telemetry.", + "processing-settings-hint": "Define how incoming messages are processed. In Basic mode, select a preconfigured processing strategy or enable only WebSocket updates. Advanced mode allows you to select individual processing strategies for each action.", + "advanced-settings-hint": "Be cautious when configuring processing strategies. Certain combinations can lead to unexpected behavior.", "strategy": "Strategy", "deduplication-interval": "Deduplication interval", "deduplication-interval-required": "Deduplication interval is required", @@ -5148,7 +5148,7 @@ "web-sockets-only": "WebSockets only" }, "time-series": "Time series", - "latest": "Latest", + "latest": "Latest values", "web-sockets": "WebSockets" }, "key-val": { From cc6f0b6c7b5a1f560b70b24efd984ec6decbda35 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 7 Feb 2025 15:59:52 +0200 Subject: [PATCH 223/416] UI: Value stepper widget --- .../json/system/widget_bundles/buttons.json | 1 + .../widget_bundles/control_widgets.json | 1 + .../system/widget_types/value_stepper.json | 51 +++ .../basic/basic-widget-config.module.ts | 5 + .../power-button-basic-config.component.html | 32 ++ .../power-button-basic-config.component.ts | 38 +- .../value-stepper-basic-config.component.html | 297 ++++++++++++++ .../value-stepper-basic-config.component.ts | 225 ++++++++++ .../lib/rpc/power-button-widget.component.ts | 7 +- .../lib/rpc/power-button-widget.models.ts | 373 +++++++++++------ .../rpc/value-stepper-widget.component.html | 43 ++ .../rpc/value-stepper-widget.component.scss | 90 ++++ .../lib/rpc/value-stepper-widget.component.ts | 388 ++++++++++++++++++ .../lib/rpc/value-stepper-widget.models.ts | 239 +++++++++++ ...lue-stepper-widget-settings.component.html | 266 ++++++++++++ ...value-stepper-widget-settings.component.ts | 191 +++++++++ .../lib/settings/widget-settings.module.ts | 5 + .../widget/widget-components.module.ts | 3 + .../assets/locale/locale.constant-en_US.json | 37 ++ .../assets/widget/value-stepper/filled.svg | 1 + .../widget/value-stepper/simplified.svg | 1 + .../assets/widget/value-stepper/volume.svg | 1 + 22 files changed, 2165 insertions(+), 130 deletions(-) create mode 100644 application/src/main/data/json/system/widget_types/value_stepper.json create mode 100644 ui-ngx/src/app/modules/home/components/widget/config/basic/rpc/value-stepper-basic-config.component.html create mode 100644 ui-ngx/src/app/modules/home/components/widget/config/basic/rpc/value-stepper-basic-config.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.html create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.models.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/settings/control/value-stepper-widget-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/settings/control/value-stepper-widget-settings.component.ts create mode 100644 ui-ngx/src/assets/widget/value-stepper/filled.svg create mode 100644 ui-ngx/src/assets/widget/value-stepper/simplified.svg create mode 100644 ui-ngx/src/assets/widget/value-stepper/volume.svg diff --git a/application/src/main/data/json/system/widget_bundles/buttons.json b/application/src/main/data/json/system/widget_bundles/buttons.json index facca89336..939e583686 100644 --- a/application/src/main/data/json/system/widget_bundles/buttons.json +++ b/application/src/main/data/json/system/widget_bundles/buttons.json @@ -12,6 +12,7 @@ "command_button", "toggle_button", "two_segment_button", + "value_stepper", "power_button" ] } \ No newline at end of file diff --git a/application/src/main/data/json/system/widget_bundles/control_widgets.json b/application/src/main/data/json/system/widget_bundles/control_widgets.json index c3dc219f5b..71a25df0c2 100644 --- a/application/src/main/data/json/system/widget_bundles/control_widgets.json +++ b/application/src/main/data/json/system/widget_bundles/control_widgets.json @@ -12,6 +12,7 @@ "command_button", "toggle_button", "two_segment_button", + "value_stepper", "power_button", "slider", "control_widgets.switch_control", diff --git a/application/src/main/data/json/system/widget_types/value_stepper.json b/application/src/main/data/json/system/widget_types/value_stepper.json new file mode 100644 index 0000000000..0126b25470 --- /dev/null +++ b/application/src/main/data/json/system/widget_types/value_stepper.json @@ -0,0 +1,51 @@ +{ + "fqn": "slider", + "name": "Value stepper", + "deprecated": false, + "image": "tb-image;/api/images/system/value-stepper-widget.svg", + "description": "Allows users to click the buttons to send commands to devices or update attributes/time series data. Configurable settings let users define how to retrieve the initial state and specify actions for each button.", + "descriptor": { + "type": "rpc", + "sizeX": 3.5, + "sizeY": 2, + "resources": [], + "templateHtml": "\n", + "templateCss": "", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.actionWidget.onInit();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '230px',\n previewHeight: '110px',\n embedTitlePanel: true,\n displayRpcMessageToast: false\n };\n};\n\nself.onDestroy = function() {\n}\n", + "dataKeySettingsForm": [], + "settingsDirective": "tb-value-stepper-widget-settings", + "hasBasicMode": true, + "basicModeDirective": "tb-value-stepper-basic-config", + "defaultConfig": "{\"showTitle\":true,\"backgroundColor\":\"#ffffff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"0px\",\"settings\":{\"initialState\":{\"action\":\"EXECUTE_RPC\",\"defaultValue\":0,\"executeRpc\":{\"method\":\"getState\",\"requestTimeout\":5000,\"requestPersistent\":false,\"persistentPollingInterval\":1000},\"getAttribute\":{\"key\":\"state\",\"scope\":null},\"getTimeSeries\":{\"key\":\"state\"},\"getAlarmStatus\":{\"severityList\":null,\"typeList\":null},\"dataToValue\":{\"type\":\"NONE\",\"compareToValue\":true,\"dataToValueFunction\":\"/* Should return integer value */\\nreturn data;\"}},\"disabledState\":{\"action\":\"DO_NOTHING\",\"defaultValue\":false,\"getAttribute\":{\"key\":\"state\",\"scope\":null},\"getTimeSeries\":{\"key\":\"state\"},\"getAlarmStatus\":{\"severityList\":null,\"typeList\":null},\"dataToValue\":{\"type\":\"NONE\",\"compareToValue\":true,\"dataToValueFunction\":\"/* Should return boolean value */\\nreturn data;\"}},\"leftButtonClick\":{\"action\":\"EXECUTE_RPC\",\"executeRpc\":{\"method\":\"setState\",\"requestTimeout\":5000,\"requestPersistent\":false,\"persistentPollingInterval\":1000},\"setAttribute\":{\"key\":\"state\",\"scope\":\"SERVER_SCOPE\"},\"putTimeSeries\":{\"key\":\"state\"},\"valueToData\":{\"type\":\"VALUE\",\"constantValue\":0,\"valueToDataFunction\":\"/* Convert input integer value to RPC parameters or attribute/time-series value */\\nreturn value;\"}},\"rightButtonClick\":{\"action\":\"EXECUTE_RPC\",\"executeRpc\":{\"method\":\"setState\",\"requestTimeout\":5000,\"requestPersistent\":false,\"persistentPollingInterval\":1000},\"setAttribute\":{\"key\":\"state\",\"scope\":\"SERVER_SCOPE\"},\"putTimeSeries\":{\"key\":\"state\"},\"valueToData\":{\"type\":\"VALUE\",\"constantValue\":0,\"valueToDataFunction\":\"/* Convert input integer value to RPC parameters or attribute/time-series value */\\nreturn value;\"}},\"appearance\":{\"type\":\"simplified\",\"autoScale\":true,\"minValueRange\":-100,\"maxValueRange\":100,\"valueStep\":0.5,\"showValueBox\":true,\"valueUnits\":\"\",\"valueDecimals\":1,\"valueFont\":{\"family\":\"Roboto\",\"weight\":\"500\",\"style\":\"normal\",\"size\":16,\"sizeUnit\":\"px\",\"lineHeight\":\"24px\"},\"valueColor\":\"#000\",\"valueBoxBackground\":\"rgba(0, 0, 0, 0.04)\",\"showBorder\":true,\"borderWidth\":1,\"borderColor\":\"#305680\"},\"buttonAppearance\":{\"leftButton\":{\"showButton\":true,\"icon\":\"arrow_back_ios_new\",\"iconSize\":24,\"iconSizeUnit\":\"px\",\"mainColorOn\":\"#3F52DD\",\"backgroundColorOn\":\"#FFFFFF\",\"mainColorDisabled\":\"rgba(0,0,0,0.12)\",\"backgroundColorDisabled\":\"#FFFFFF\",\"customStyle\":{\"enabled\":null,\"hovered\":null,\"pressed\":null,\"activated\":null,\"disabled\":null}},\"rightButton\":{\"showButton\":true,\"icon\":\"arrow_forward_ios\",\"iconSize\":24,\"iconSizeUnit\":\"px\",\"mainColorOn\":\"#3F52DD\",\"backgroundColorOn\":\"#FFFFFF\",\"mainColorDisabled\":\"rgba(0,0,0,0.12)\",\"backgroundColorDisabled\":\"#FFFFFF\",\"customStyle\":{\"enabled\":null,\"hovered\":null,\"pressed\":null,\"activated\":null,\"disabled\":null}}},\"background\":{\"type\":\"color\",\"color\":\"#fff\",\"overlay\":{\"enabled\":false,\"color\":\"rgba(255,255,255,0.72)\",\"blur\":3}},\"padding\":\"12px\"},\"title\":\"Value stepper\",\"dropShadow\":true,\"enableFullscreen\":false,\"widgetStyle\":{},\"actions\":{},\"widgetCss\":\"\",\"noDataDisplayMessage\":\"\",\"titleFont\":{\"size\":16,\"sizeUnit\":\"px\",\"family\":null,\"weight\":\"500\",\"style\":null,\"lineHeight\":\"24px\"},\"showTitleIcon\":false,\"titleTooltip\":\"\",\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"pageSize\":1024,\"titleIcon\":\"\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"14px\",\"configMode\":\"basic\",\"titleColor\":\"rgba(0, 0, 0, 0.87)\",\"datasources\":null,\"borderRadius\":null}" + }, + "resources": [ + { + "link": "/api/images/system/value-stepper-widget.svg", + "title": "\"Value stepper\" system widget image", + "type": "IMAGE", + "subType": "IMAGE", + "fileName": "value-stepper-widget.svg", + "publicResourceKey": "s0UKoqbiMCcKVn0pD55XZzPUR89XlXAO", + "mediaType": "image/svg+xml", + "data": "PHN2ZyB3aWR0aD0iMjE0IiBoZWlnaHQ9Ijc2IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjxnIGZpbHRlcj0idXJsKCNhKSI+PHJlY3QgeD0iOC41IiB5PSI0LjUiIHdpZHRoPSIxOTciIGhlaWdodD0iNTkiIHJ4PSI0IiBmaWxsPSIjZmZmIiBzaGFwZS1yZW5kZXJpbmc9ImNyaXNwRWRnZXMiLz48cmVjdCB4PSI5IiB5PSI1IiB3aWR0aD0iMTk2IiBoZWlnaHQ9IjU4IiByeD0iMy41IiBzdHJva2U9IiMzMDU2ODAiIHNoYXBlLXJlbmRlcmluZz0iY3Jpc3BFZGdlcyIvPjxyZWN0IHg9IjIwLjUiIHk9IjE4IiB3aWR0aD0iMzIiIGhlaWdodD0iMzIiIHJ4PSIxNiIgZmlsbD0iIzMwNTY4MCIvPjxwYXRoIGQ9Im0zOC41IDQwIDEuNDEtMS40MUwzNS4zMyAzNGw0LjU4LTQuNTlMMzguNSAyOGwtNiA2IDYgNloiIGZpbGw9IiNmZmYiLz48cmVjdCB4PSI2NSIgeT0iMTguNSIgd2lkdGg9Ijg0IiBoZWlnaHQ9IjMxIiByeD0iMy41IiBmaWxsPSIjMzA1NjgwIiBmaWxsLW9wYWNpdHk9Ii4wNCIvPjxyZWN0IHg9IjY1IiB5PSIxOC41IiB3aWR0aD0iODQiIGhlaWdodD0iMzEiIHJ4PSIzLjUiIHN0cm9rZT0iIzMwNTY4MCIvPjxwYXRoIGQ9Ik04OC41OSAzNy41VjM5aC03LjYzdi0xLjI5bDMuNy00LjA0Yy40MS0uNDYuNzMtLjg1Ljk3LTEuMTkuMjMtLjMzLjQtLjYzLjQ5LS45YTIuMyAyLjMgMCAwIDAtLjA2LTEuNzNjLS4xMy0uMjctLjMyLS41LS41OC0uNjVhMS43IDEuNyAwIDAgMC0uOTMtLjI0Yy0uNDIgMC0uNzcuMS0xLjA2LjI3LS4yOC4xOS0uNS40NC0uNjUuNzZhMi42IDIuNiAwIDAgMC0uMjIgMS4xaC0xLjg4YzAtLjY3LjE1LTEuMjcuNDYtMS44Mi4zLS41NS43My0uOTkgMS4zLTEuM2E0LjEgNC4xIDAgMCAxIDIuMDgtLjVjLjc2IDAgMS40LjEzIDEuOTQuMzguNTMuMjYuOTMuNjIgMS4yIDEuMDlhMy4zNiAzLjM2IDAgMCAxIC4yNSAyLjcyIDUgNSAwIDAgMS0uNDkgMS4wNCA5IDkgMCAwIDEtLjc0IDEuMDRjLS4yOC4zNS0uNi43LS45NCAxLjA1bC0yLjQ2IDIuNzFoNS4yNVptOS4yNy05Ljg4djEuMDRMOTMuMyAzOWgtMS45OGw0LjU0LTkuODhoLTUuOXYtMS41aDcuODlabTEuOTggMTAuNDRjMC0uMjkuMS0uNTMuMy0uNzMuMi0uMi40Ni0uMy44LS4zcy42LjEuOC4zYy4yLjIuMy40NC4zLjczYTEgMSAwIDAgMS0uMy43NGMtLjIuMi0uNDYuMy0uOC4zcy0uNi0uMS0uOC0uM2ExIDEgMCAwIDEtLjMtLjc0Wm02LjQ5LTQuMzUtMS41LS4zNy42MS01LjcyaDYuMTR2MS42SDEwN2wtLjMyIDIuNzlhMy42NyAzLjY3IDAgMCAxIDEuODEtLjQ2Yy41NCAwIDEuMDIuMDkgMS40NC4yNi40Mi4xNy43OS40MyAxLjA4Ljc2LjMuMzMuNTMuNzMuNjggMS4yYTUgNSAwIDAgMSAwIDMuMDcgMy4xNyAzLjE3IDAgMCAxLTEuODUgMi4wM2MtLjQ2LjE5LTEuMDEuMjktMS42NS4yOWE0LjYgNC42IDAgMCAxLTEuMzYtLjIgMy43MyAzLjczIDAgMCAxLTEuMTctLjYyIDMuMTQgMy4xNCAwIDAgMS0xLjE5LTIuNDJoMS44NWMuMDUuMzcuMTUuNjkuMy45NS4xNi4yNS4zOC40NS42NC41OGEyIDIgMCAwIDAgLjkyLjJjLjMyIDAgLjYtLjA1LjgzLS4xNi4yMy0uMTEuNDItLjI3LjU3LS40OC4xNS0uMjIuMjctLjQ3LjM0LS43NWEzLjYyIDMuNjIgMCAwIDAtLjAyLTEuODcgMS45OCAxLjk4IDAgMCAwLS4zOC0uNzJjLS4xNy0uMi0uMzgtLjM2LS42My0uNDdhMi4xMyAyLjEzIDAgMCAwLS44OC0uMTdjLS40NSAwLS44LjA3LTEuMDQuMi0uMjMuMTMtLjQ1LjI5LS42NS40OFptMTEuNzUtNC4xNmMwLS4zOC4xLS43Mi4yOC0xLjA0LjE5LS4zMi40NC0uNTcuNzUtLjc2YTEuOTYgMS45NiAwIDAgMSAyLjA1IDBjLjMxLjE5LjU2LjQ0Ljc0Ljc2LjE5LjMyLjI4LjY2LjI4IDEuMDRzLS4xLjczLS4yOCAxLjA1Yy0uMTguMzEtLjQzLjU2LS43NC43NGEyLjA0IDIuMDQgMCAwIDEtMi44LS43NCAyLjAzIDIuMDMgMCAwIDEtLjI4LTEuMDVabTEuMDUgMGExIDEgMCAwIDAgMSAxIC45Ny45NyAwIDAgMCAuOTgtMSAxIDEgMCAwIDAtLjI3LS43Mi45My45MyAwIDAgMC0uNy0uM2MtLjI4IDAtLjUxLjEtLjcxLjMtLjIuMi0uMy40My0uMy43MlptMTIuMTkgNS43NWgxLjk1YTQuNSA0LjUgMCAwIDEtLjYyIDEuOTkgMy43MiAzLjcyIDAgMCAxLTEuNSAxLjM3IDUgNSAwIDAgMS0yLjMzLjUgNC4xNSA0LjE1IDAgMCAxLTMuMzQtMS40NWMtLjQtLjQ4LS43MS0xLjA0LS45Mi0xLjctLjIxLS42Ni0uMzItMS40LS4zMi0yLjIydi0uOTVjMC0uODEuMS0xLjU1LjMyLTIuMjIuMjItLjY2LjUzLTEuMjIuOTQtMS42OS40LS40Ny45LS44NCAxLjQ2LTEuMDlhNC43OCA0Ljc4IDAgMCAxIDEuOTMtLjM3Yy45IDAgMS42Ny4xNyAyLjMuNS42Mi4zMyAxLjEuOCAxLjQ1IDEuMzguMzUuNTkuNTYgMS4yNi42NCAyLjAyaC0xLjk1Yy0uMDUtLjQ4LS4xNy0uOS0uMzUtMS4yNWExLjc3IDEuNzcgMCAwIDAtLjc2LS44IDIuNzMgMi43MyAwIDAgMC0xLjMzLS4yOGMtLjQ1IDAtLjg0LjA4LTEuMTcuMjVhMi4yIDIuMiAwIDAgMC0uODQuNzNjLS4yMi4zMy0uMzkuNzItLjUgMS4yLS4xMS40Ny0uMTcgMS0uMTcgMS42di45N2MwIC41Ny4wNSAxLjEuMTUgMS41Ni4xLjQ3LjI2Ljg2LjQ3IDEuMi4yMS4zMy40OC41OS44MS43Ny4zMy4xOC43Mi4yNyAxLjE4LjI3LjU2IDAgMS0uMDggMS4zNS0uMjYuMzUtLjE4LjYxLS40NC44LS43OC4xNy0uMzQuMy0uNzYuMzUtMS4yNVoiIGZpbGw9IiMwMDAiIGZpbGwtb3BhY2l0eT0iLjg3Ii8+PHJlY3QgeD0iMTYxLjUiIHk9IjE4IiB3aWR0aD0iMzIiIGhlaWdodD0iMzIiIHJ4PSIxNiIgZmlsbD0iIzMwNTY4MCIvPjxwYXRoIGQ9Im0xNzUuNSAyOC0xLjQxIDEuNDEgNC41OCA0LjU5LTQuNTggNC41OUwxNzUuNSA0MGw2LTYtNi02WiIgZmlsbD0iI2ZmZiIvPjwvZz48ZGVmcz48ZmlsdGVyIGlkPSJhIiB4PSIuNSIgeT0iLjUiIHdpZHRoPSIyMTMiIGhlaWdodD0iNzUiIGZpbHRlclVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgY29sb3ItaW50ZXJwb2xhdGlvbi1maWx0ZXJzPSJzUkdCIj48ZmVGbG9vZCBmbG9vZC1vcGFjaXR5PSIwIiByZXN1bHQ9IkJhY2tncm91bmRJbWFnZUZpeCIvPjxmZUNvbG9yTWF0cml4IGluPSJTb3VyY2VBbHBoYSIgdmFsdWVzPSIwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAxMjcgMCIgcmVzdWx0PSJoYXJkQWxwaGEiLz48ZmVPZmZzZXQgZHk9IjQiLz48ZmVHYXVzc2lhbkJsdXIgc3RkRGV2aWF0aW9uPSI0Ii8+PGZlQ29tcG9zaXRlIGluMj0iaGFyZEFscGhhIiBvcGVyYXRvcj0ib3V0Ii8+PGZlQ29sb3JNYXRyaXggdmFsdWVzPSIwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwLjA0IDAiLz48ZmVCbGVuZCBpbjI9IkJhY2tncm91bmRJbWFnZUZpeCIgcmVzdWx0PSJlZmZlY3QxX2Ryb3BTaGFkb3dfNTY2OV8xNjA3MDUiLz48ZmVCbGVuZCBpbj0iU291cmNlR3JhcGhpYyIgaW4yPSJlZmZlY3QxX2Ryb3BTaGFkb3dfNTY2OV8xNjA3MDUiIHJlc3VsdD0ic2hhcGUiLz48L2ZpbHRlcj48L2RlZnM+PC9zdmc+", + "public": true + } + ], + "scada": false, + "tags": [ + "command", + "downlink", + "device configuration", + "device control", + "invocation", + "remote method", + "remote function", + "interface", + "subroutine call", + "inter-process communication", + "server request", + "update attribute", + "set attribute", + "add time-series" + ] +} \ No newline at end of file diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts index 24cb04cca6..8a71e04e1e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts @@ -146,6 +146,9 @@ import { ScadaSymbolBasicConfigComponent } from '@home/components/widget/config/ import { SegmentedButtonBasicConfigComponent } from '@home/components/widget/config/basic/button/segmented-button-basic-config.component'; +import { + ValueStepperBasicConfigComponent +} from '@home/components/widget/config/basic/rpc/value-stepper-basic-config.component'; @NgModule({ declarations: [ @@ -182,6 +185,7 @@ import { PowerButtonBasicConfigComponent, SliderBasicConfigComponent, ToggleButtonBasicConfigComponent, + ValueStepperBasicConfigComponent, TimeSeriesChartBasicConfigComponent, ComparisonKeyRowComponent, ComparisonKeysTableComponent, @@ -236,6 +240,7 @@ import { PowerButtonBasicConfigComponent, SliderBasicConfigComponent, ToggleButtonBasicConfigComponent, + ValueStepperBasicConfigComponent, TimeSeriesChartBasicConfigComponent, StatusWidgetBasicConfigComponent, PieChartBasicConfigComponent, diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/button/power-button-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/button/power-button-basic-config.component.html index 75fcf791b0..a2e3f90aee 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/button/power-button-basic-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/button/power-button-basic-config.component.html @@ -116,6 +116,38 @@

        +
        + + {{ 'widgets.power-button.button-icon-on' | translate }} + +
        + + + + + + +
        +
        +
        + + {{ 'widgets.power-button.button-icon-off' | translate }} + +
        + + + + + + +
        +
        {{ 'widgets.power-button.power-on-colors' | translate }}
        diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/button/power-button-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/button/power-button-basic-config.component.ts index 2c92936141..00b0a41e18 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/button/power-button-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/button/power-button-basic-config.component.ts @@ -87,6 +87,19 @@ export class PowerButtonBasicConfigComponent extends BasicWidgetConfigComponent icon: [configData.config.titleIcon, []], iconColor: [configData.config.iconColor, []], + onButtonIcon: this.fb.group({ + showIcon: [settings.onButtonIcon.showIcon, []], + iconSize: [settings.onButtonIcon.iconSize, [Validators.min(0)]], + iconSizeUnit: [settings.onButtonIcon.iconSizeUnit, []], + icon: [settings.onButtonIcon.icon, []], + }), + offButtonIcon: this.fb.group({ + showIcon: [settings.offButtonIcon.showIcon, []], + iconSize: [settings.offButtonIcon.iconSize, [Validators.min(0)]], + iconSizeUnit: [settings.offButtonIcon.iconSizeUnit, []], + icon: [settings.offButtonIcon.icon, []], + }), + mainColorOn: [settings.mainColorOn, []], backgroundColorOn: [settings.backgroundColorOn, []], @@ -128,6 +141,9 @@ export class PowerButtonBasicConfigComponent extends BasicWidgetConfigComponent this.widgetConfig.config.settings.layout = config.layout; + this.widgetConfig.config.settings.onButtonIcon = config.onButtonIcon; + this.widgetConfig.config.settings.offButtonIcon = config.offButtonIcon; + this.widgetConfig.config.settings.mainColorOn = config.mainColorOn; this.widgetConfig.config.settings.backgroundColorOn = config.backgroundColorOn; @@ -148,12 +164,14 @@ export class PowerButtonBasicConfigComponent extends BasicWidgetConfigComponent } protected validatorTriggers(): string[] { - return ['showTitle', 'showIcon']; + return ['showTitle', 'showIcon', 'onButtonIcon.showIcon', 'offButtonIcon.showIcon']; } protected updateValidators(emitEvent: boolean, trigger?: string) { const showTitle: boolean = this.powerButtonWidgetConfigForm.get('showTitle').value; const showIcon: boolean = this.powerButtonWidgetConfigForm.get('showIcon').value; + const onButtonIcon: boolean = this.powerButtonWidgetConfigForm.get('onButtonIcon').get('showIcon').value; + const offButtonIcon: boolean = this.powerButtonWidgetConfigForm.get('offButtonIcon').get('showIcon').value; if (showTitle) { this.powerButtonWidgetConfigForm.get('title').enable(); this.powerButtonWidgetConfigForm.get('titleFont').enable(); @@ -180,6 +198,24 @@ export class PowerButtonBasicConfigComponent extends BasicWidgetConfigComponent this.powerButtonWidgetConfigForm.get('icon').disable(); this.powerButtonWidgetConfigForm.get('iconColor').disable(); } + if (onButtonIcon) { + this.powerButtonWidgetConfigForm.get('onButtonIcon').get('iconSize').enable(); + this.powerButtonWidgetConfigForm.get('onButtonIcon').get('iconSizeUnit').enable(); + this.powerButtonWidgetConfigForm.get('onButtonIcon').get('icon').enable(); + } else { + this.powerButtonWidgetConfigForm.get('onButtonIcon').get('iconSize').disable(); + this.powerButtonWidgetConfigForm.get('onButtonIcon').get('iconSizeUnit').disable(); + this.powerButtonWidgetConfigForm.get('onButtonIcon').get('icon').disable(); + } + if (offButtonIcon) { + this.powerButtonWidgetConfigForm.get('offButtonIcon').get('iconSize').enable(); + this.powerButtonWidgetConfigForm.get('offButtonIcon').get('iconSizeUnit').enable(); + this.powerButtonWidgetConfigForm.get('offButtonIcon').get('icon').enable(); + } else { + this.powerButtonWidgetConfigForm.get('offButtonIcon').get('iconSize').disable(); + this.powerButtonWidgetConfigForm.get('offButtonIcon').get('iconSizeUnit').disable(); + this.powerButtonWidgetConfigForm.get('offButtonIcon').get('icon').disable(); + } } private getCardButtons(config: WidgetConfig): string[] { diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/rpc/value-stepper-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/rpc/value-stepper-basic-config.component.html new file mode 100644 index 0000000000..748cf64c7f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/rpc/value-stepper-basic-config.component.html @@ -0,0 +1,297 @@ + + + +
        +
        widgets.value-stepper.behavior
        +
        +
        widgets.value-stepper.initial-state
        + +
        +
        +
        widgets.value-stepper.left-button-click
        + +
        +
        +
        widgets.value-stepper.right-button-click
        + +
        +
        +
        widgets.button-state.disabled-state
        + +
        +
        +
        +
        widget-config.appearance
        + + + {{ valueStepperTypeTranslationMap.get(type) | translate }} + + +
        + + {{ 'widgets.value-stepper.auto-scale' | translate }} + +
        + +
        +
        {{ 'widgets.value-stepper.value-range' | translate }}
        +
        +
        widgets.value-stepper.min-range
        + + + +
        widgets.value-stepper.max-range
        + + + +
        +
        +
        +
        {{ 'widgets.value-stepper.value-increment-decrement-step' | translate }}
        + + + +
        +
        + + {{ 'widgets.value-stepper.value' | translate }} + +
        + + + +
        widget-config.decimals-suffix
        +         + + + + +
        +
        + +
        +
        {{ 'widgets.value-stepper.value-box-background' | translate }}
        + + +
        +
        + + {{ 'widgets.value-stepper.border' | translate }} + +
        + + +
        px
        +         + + +
        +
        +
        +
        +
        +
        widgets.value-stepper.button-appearance
        + + {{ 'widgets.value-stepper.left' | translate }} + {{ 'widgets.value-stepper.right' | translate }} + +
        +
        +
        + + {{ 'widgets.value-stepper.left-button' | translate }} + +
        +
        +
        {{ 'widgets.value-stepper.icon' | translate }}
        +
        + + + + + + +
        +
        +
        +
        {{ 'widgets.value-stepper.button-on-colors' | translate }}
        +
        +
        +
        widgets.value-stepper.main
        + + +
        + +
        +
        widgets.value-stepper.background
        + + +
        +
        +
        +
        +
        {{ 'widgets.value-stepper.disabled-colors' | translate }}
        +
        +
        +
        widgets.value-stepper.main
        + + +
        + +
        +
        widgets.value-stepper.background
        + + +
        +
        +
        +
        +
        +
        + + {{ 'widgets.value-stepper.right-button' | translate }} + +
        +
        +
        {{ 'widgets.value-stepper.icon' | translate }}
        +
        + + + + + + +
        +
        +
        +
        {{ 'widgets.value-stepper.button-on-colors' | translate }}
        +
        +
        +
        widgets.value-stepper.main
        + + +
        + +
        +
        widgets.value-stepper.background
        + + +
        +
        +
        +
        +
        {{ 'widgets.value-stepper.disabled-colors' | translate }}
        +
        +
        +
        widgets.value-stepper.main
        + + +
        + +
        +
        widgets.value-stepper.background
        + + +
        +
        +
        +
        +
        +
        +
        widget-config.card-appearance
        +
        +
        {{ 'widgets.background.background' | translate }}
        + + +
        +
        +
        widget-config.show-card-buttons
        + + {{ 'fullscreen.fullscreen' | translate }} + +
        +
        +
        {{ 'widget-config.card-border-radius' | translate }}
        + + + +
        +
        +
        {{ 'widget-config.card-padding' | translate }}
        + + + +
        +
        + + +         diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/rpc/value-stepper-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/rpc/value-stepper-basic-config.component.ts new file mode 100644 index 0000000000..abee223b40 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/rpc/value-stepper-basic-config.component.ts @@ -0,0 +1,225 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component } from '@angular/core'; +import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { BasicWidgetConfigComponent } from '@home/components/widget/config/widget-config.component.models'; +import { WidgetConfigComponentData } from '@home/models/widget-component.models'; +import { TargetDevice, WidgetConfig, } from '@shared/models/widget.models'; +import { WidgetConfigComponent } from '@home/components/widget/widget-config.component'; +import { formatValue, isUndefined } from '@core/utils'; +import { ValueType } from '@shared/models/constants'; +import { + valueStepperDefaultSettings, + valueStepperTypeImages, + valueStepperTypes, + valueStepperTypeTranslations, + ValueStepperWidgetSettings +} from '@home/components/widget/lib/rpc/value-stepper-widget.models'; + +type ButtonAppearanceType = 'left' | 'right'; + +@Component({ + selector: 'tb-value-stepper-basic-config', + templateUrl: './value-stepper-basic-config.component.html', + styleUrls: ['../basic-config.scss'] +}) +export class ValueStepperBasicConfigComponent extends BasicWidgetConfigComponent { + + get targetDevice(): TargetDevice { + return this.valueStepperWidgetConfigForm.get('targetDevice').value; + } + + valueStepperTypeTranslationMap = valueStepperTypeTranslations; + valueStepperTypes = valueStepperTypes; + valueStepperTypeImageMap = valueStepperTypeImages; + + buttonAppearanceType: ButtonAppearanceType = 'left'; + + valueType = ValueType; + + valueStepperWidgetConfigForm: UntypedFormGroup; + + valuePreviewFn = this._valuePreviewFn.bind(this); + + constructor(protected store: Store, + protected widgetConfigComponent: WidgetConfigComponent, + private fb: UntypedFormBuilder) { + super(store, widgetConfigComponent); + } + + protected configForm(): UntypedFormGroup { + return this.valueStepperWidgetConfigForm; + } + + protected onConfigSet(configData: WidgetConfigComponentData) { + const settings: ValueStepperWidgetSettings = {...valueStepperDefaultSettings, ...(configData.config.settings || {})}; + this.valueStepperWidgetConfigForm = this.fb.group({ + targetDevice: [configData.config.targetDevice, []], + + initialState: [settings.initialState, []], + leftButtonClick: [settings.leftButtonClick, []], + rightButtonClick: [settings.rightButtonClick, []], + disabledState: [settings.disabledState, []], + + appearance: this.fb.group({ + type: [settings.appearance.type, []], + autoScale: [settings.appearance.autoScale, []], + minValueRange: [settings.appearance.minValueRange, []], + maxValueRange: [settings.appearance.maxValueRange, []], + valueStep: [settings.appearance.valueStep, [Validators.min(0)]], + showValueBox: [settings.appearance.showValueBox, []], + valueUnits: [settings.appearance.valueUnits, []], + valueDecimals: [settings.appearance.valueDecimals, []], + valueFont: [settings.appearance.valueFont, []], + valueColor: [settings.appearance.valueColor], + valueBoxBackground: [settings.appearance.valueBoxBackground, []], + showBorder: [settings.appearance.showBorder, []], + borderWidth: [settings.appearance.borderWidth, []], + borderColor: [settings.appearance.borderColor, []] + }), + + buttonAppearance: this.fb.group({ + leftButton: this.fb.group({ + showButton: [settings.buttonAppearance.leftButton.showButton], + icon: [settings.buttonAppearance.leftButton.icon], + iconSize: [settings.buttonAppearance.leftButton.iconSize], + iconSizeUnit: [settings.buttonAppearance.leftButton.iconSizeUnit], + mainColorOn: [settings.buttonAppearance.leftButton.mainColorOn, []], + backgroundColorOn: [settings.buttonAppearance.leftButton.backgroundColorOn, []], + mainColorDisabled: [settings.buttonAppearance.leftButton.mainColorDisabled, []], + backgroundColorDisabled: [settings.buttonAppearance.leftButton.backgroundColorDisabled, []] + }), + rightButton: this.fb.group({ + showButton: [settings.buttonAppearance.rightButton.showButton], + icon: [settings.buttonAppearance.rightButton.icon], + iconSize: [settings.buttonAppearance.rightButton.iconSize], + iconSizeUnit: [settings.buttonAppearance.rightButton.iconSizeUnit], + mainColorOn: [settings.buttonAppearance.rightButton.mainColorOn, []], + backgroundColorOn: [settings.buttonAppearance.rightButton.backgroundColorOn, []], + mainColorDisabled: [settings.buttonAppearance.rightButton.mainColorDisabled, []], + backgroundColorDisabled: [settings.buttonAppearance.rightButton.backgroundColorDisabled, []] + }) + }), + + background: [settings.background, []], + cardButtons: [this.getCardButtons(configData.config), []], + borderRadius: [configData.config.borderRadius, []], + padding: [settings.padding, []], + + actions: [configData.config.actions || {}, []] + }); + } + + + protected prepareOutputConfig(config: any): WidgetConfigComponentData { + this.widgetConfig.config.targetDevice = config.targetDevice; + this.widgetConfig.config.settings = this.widgetConfig.config.settings || {}; + this.widgetConfig.config.settings.initialState = config.initialState; + this.widgetConfig.config.settings.disabledState = config.disabledState; + this.widgetConfig.config.settings.leftButtonClick = config.leftButtonClick; + this.widgetConfig.config.settings.rightButtonClick = config.rightButtonClick; + this.widgetConfig.config.settings.appearance = config.appearance; + this.widgetConfig.config.settings.buttonAppearance = config.buttonAppearance; + + this.widgetConfig.config.settings.background = config.background; + this.setCardButtons(config.cardButtons, this.widgetConfig.config); + this.widgetConfig.config.borderRadius = config.borderRadius; + this.widgetConfig.config.settings.padding = config.padding; + this.widgetConfig.config.actions = config.actions; + + return this.widgetConfig; + } + + + protected validatorTriggers(): string[] { + return ['appearance.showValueBox', 'appearance.showBorder', + 'buttonAppearance.leftButton.showButton', 'buttonAppearance.rightButton.showButton']; + } + + protected updateValidators(emitEvent: boolean, trigger?: string) { + const showValueBox: boolean = this.valueStepperWidgetConfigForm.get('appearance').get('showValueBox').value; + const showBorder: boolean = this.valueStepperWidgetConfigForm.get('appearance').get('showBorder').value; + const showLeftButton: boolean = this.valueStepperWidgetConfigForm.get('buttonAppearance').get('leftButton').get('showButton').value; + const showRightButton: boolean = this.valueStepperWidgetConfigForm.get('buttonAppearance').get('rightButton').get('showButton').value; + if (showValueBox) { + this.valueStepperWidgetConfigForm.get('appearance').get('valueUnits').enable(); + this.valueStepperWidgetConfigForm.get('appearance').get('valueDecimals').enable(); + this.valueStepperWidgetConfigForm.get('appearance').get('valueFont').enable(); + this.valueStepperWidgetConfigForm.get('appearance').get('valueColor').enable(); + this.valueStepperWidgetConfigForm.get('appearance').get('valueBoxBackground').enable(); + this.valueStepperWidgetConfigForm.get('appearance').get('showBorder').enable({emitEvent: false}); + if (showBorder) { + this.valueStepperWidgetConfigForm.get('appearance').get('borderWidth').enable(); + this.valueStepperWidgetConfigForm.get('appearance').get('borderColor').enable(); + } else { + this.valueStepperWidgetConfigForm.get('appearance').get('borderWidth').disable(); + this.valueStepperWidgetConfigForm.get('appearance').get('borderColor').disable(); + } + } else { + this.valueStepperWidgetConfigForm.get('appearance').get('valueUnits').disable(); + this.valueStepperWidgetConfigForm.get('appearance').get('valueDecimals').disable(); + this.valueStepperWidgetConfigForm.get('appearance').get('valueFont').disable(); + this.valueStepperWidgetConfigForm.get('appearance').get('valueColor').disable(); + this.valueStepperWidgetConfigForm.get('appearance').get('valueBoxBackground').disable(); + this.valueStepperWidgetConfigForm.get('appearance').get('showBorder').disable({emitEvent: false}); + this.valueStepperWidgetConfigForm.get('appearance').get('borderWidth').disable(); + this.valueStepperWidgetConfigForm.get('appearance').get('borderColor').disable(); + } + this.buttonValidators(showLeftButton, 'leftButton'); + this.buttonValidators(showRightButton, 'rightButton'); + } + + private buttonValidators(showButtonValue: boolean, button: string) { + if (showButtonValue) { + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('icon').enable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('iconSize').enable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('iconSizeUnit').enable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('mainColorOn').enable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('backgroundColorOn').enable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('mainColorDisabled').enable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('backgroundColorDisabled').enable() + } else { + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('icon').disable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('iconSize').disable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('iconSizeUnit').disable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('mainColorOn').disable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('backgroundColorOn').disable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('mainColorDisabled').disable() + this.valueStepperWidgetConfigForm.get('buttonAppearance').get(button).get('backgroundColorDisabled').disable() + } + } + + private getCardButtons(config: WidgetConfig): string[] { + const buttons: string[] = []; + if (isUndefined(config.enableFullscreen) || config.enableFullscreen) { + buttons.push('fullscreen'); + } + return buttons; + } + + private setCardButtons(buttons: string[], config: WidgetConfig) { + config.enableFullscreen = buttons.includes('fullscreen'); + } + + private _valuePreviewFn(): string { + const units: string = this.valueStepperWidgetConfigForm.get('appearance').get('valueUnits').value; + const decimals: number = this.valueStepperWidgetConfigForm.get('appearance').get('valueDecimals').value; + return formatValue(48, decimals, units, false); + } +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.component.ts index 6c9072e915..262a532163 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.component.ts @@ -33,11 +33,11 @@ import { DomSanitizer } from '@angular/platform-browser'; import { ValueType } from '@shared/models/constants'; import { powerButtonDefaultSettings, - PowerButtonShape, - powerButtonShapeSize, + PowerButtonShape, powerButtonShapeSize, PowerButtonWidgetSettings } from '@home/components/widget/lib/rpc/power-button-widget.models'; import { SVG, Svg } from '@svgdotjs/svg.js'; +import { MatIconRegistry } from '@angular/material/icon'; @Component({ selector: 'tb-power-button-widget', @@ -72,6 +72,7 @@ export class PowerButtonWidgetComponent extends constructor(protected imagePipe: ImagePipe, protected sanitizer: DomSanitizer, private renderer: Renderer2, + private iconRegistry: MatIconRegistry, protected cd: ChangeDetectorRef, protected zone: NgZone) { super(cd); @@ -180,7 +181,7 @@ export class PowerButtonWidgetComponent extends this.renderer.setStyle(this.svgShape.node, 'user-select', 'none'); this.zone.run(() => { - this.powerButtonSvgShape = PowerButtonShape.fromSettings(this.ctx, this.svgShape, + this.powerButtonSvgShape = PowerButtonShape.fromSettings(this.ctx, this.svgShape, this.iconRegistry, this.settings, this.value, this.disabledState, () => this.onClick()); }); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.models.ts index 503347939d..540ef8f9f3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.models.ts @@ -28,6 +28,11 @@ import { Circle, Effect, Element, G, Gradient, Path, Runner, Svg, Text, Timeline import '@svgdotjs/svg.filter.js'; import tinycolor from 'tinycolor2'; import { WidgetContext } from '@home/models/widget-component.models'; +import { Observable, of } from 'rxjs'; +import { isSvgIcon, splitIconName } from '@shared/models/icon.models'; +import { catchError, map, take } from 'rxjs/operators'; +import { MatIconRegistry } from '@angular/material/icon'; +import { isDefinedAndNotNull } from '@core/utils'; export enum PowerButtonLayout { default = 'default', @@ -71,20 +76,29 @@ export const powerButtonLayoutImages = new Map( ] ); +export interface ButtonIconSettings { + showIcon: boolean; + iconSize: number; + iconSizeUnit: string; + icon: string; +} + export interface PowerButtonWidgetSettings { - initialState: GetValueSettings; - disabledState: GetValueSettings; - onUpdateState: SetValueSettings; - offUpdateState: SetValueSettings; + initialState?: GetValueSettings; + disabledState?: GetValueSettings; + onUpdateState?: SetValueSettings; + offUpdateState?: SetValueSettings; layout: PowerButtonLayout; + onButtonIcon: ButtonIconSettings, + offButtonIcon: ButtonIconSettings, mainColorOn: string; backgroundColorOn: string; mainColorOff: string; backgroundColorOff: string; mainColorDisabled: string; backgroundColorDisabled: string; - background: BackgroundSettings; - padding: string; + background?: BackgroundSettings; + padding?: string; } export const powerButtonDefaultSettings: PowerButtonWidgetSettings = { @@ -177,6 +191,18 @@ export const powerButtonDefaultSettings: PowerButtonWidgetSettings = { } }, layout: PowerButtonLayout.default, + onButtonIcon: { + showIcon: false, + iconSize: 32, + iconSizeUnit: 'px', + icon: 'power_settings_new' + }, + offButtonIcon: { + showIcon: false, + iconSize: 32, + iconSizeUnit: 'px', + icon: 'power_settings_new' + }, mainColorOn: '#3F52DD', backgroundColorOn: '#FFFFFF', mainColorOff: '#A2A2A2', @@ -207,6 +233,11 @@ interface PowerButtonColorState { backgroundColor: PowerButtonColor; } +interface ButtonsIconSettings { + onButtonIcon: ButtonIconSettings; + offButtonIcon: ButtonIconSettings; +} + type PowerButtonShapeColors = Record; const createPowerButtonShapeColors = (settings: PowerButtonWidgetSettings): PowerButtonShapeColors => { @@ -257,33 +288,35 @@ export abstract class PowerButtonShape { static fromSettings(ctx: WidgetContext, svgShape: Svg, + iconRegistry: MatIconRegistry, settings: PowerButtonWidgetSettings, value: boolean, disabled: boolean, onClick: () => void): PowerButtonShape { switch (settings.layout) { case PowerButtonLayout.default: - return new DefaultPowerButtonShape(ctx, svgShape, settings, value, disabled, onClick); + return new DefaultPowerButtonShape(ctx, svgShape, iconRegistry, settings, value, disabled, onClick); case PowerButtonLayout.simplified: - return new SimplifiedPowerButtonShape(ctx, svgShape, settings, value, disabled, onClick); + return new SimplifiedPowerButtonShape(ctx, svgShape, iconRegistry, settings, value, disabled, onClick); case PowerButtonLayout.outlined: - return new OutlinedPowerButtonShape(ctx, svgShape, settings, value, disabled, onClick); + return new OutlinedPowerButtonShape(ctx, svgShape, iconRegistry, settings, value, disabled, onClick); case PowerButtonLayout.default_volume: - return new DefaultVolumePowerButtonShape(ctx, svgShape, settings, value, disabled, onClick); + return new DefaultVolumePowerButtonShape(ctx, svgShape, iconRegistry, settings, value, disabled, onClick); case PowerButtonLayout.simplified_volume: - return new SimplifiedVolumePowerButtonShape(ctx, svgShape, settings, value, disabled, onClick); + return new SimplifiedVolumePowerButtonShape(ctx, svgShape, iconRegistry, settings, value, disabled, onClick); case PowerButtonLayout.outlined_volume: - return new OutlinedVolumePowerButtonShape(ctx, svgShape, settings, value, disabled, onClick); + return new OutlinedVolumePowerButtonShape(ctx, svgShape, iconRegistry, settings, value, disabled, onClick); case PowerButtonLayout.default_icon: - return new DefaultIconPowerButtonShape(ctx, svgShape, settings, value, disabled, onClick); + return new DefaultIconPowerButtonShape(ctx, svgShape, iconRegistry, settings, value, disabled, onClick); case PowerButtonLayout.simplified_icon: - return new SimplifiedIconPowerButtonShape(ctx, svgShape, settings, value, disabled, onClick); + return new SimplifiedIconPowerButtonShape(ctx, svgShape, iconRegistry, settings, value, disabled, onClick); case PowerButtonLayout.outlined_icon: - return new OutlinedIconPowerButtonShape(ctx, svgShape, settings, value, disabled, onClick); + return new OutlinedIconPowerButtonShape(ctx, svgShape, iconRegistry, settings, value, disabled, onClick); } } protected readonly colors: PowerButtonShapeColors; + protected readonly icons: ButtonsIconSettings; protected readonly onLabel: string; protected readonly offLabel: string; @@ -293,18 +326,67 @@ export abstract class PowerButtonShape { protected pressed = false; protected forcePressed = false; + protected offPowerSymbolIcon: Element; + protected onPowerSymbolIcon: Element; + protected offLabelShape: Text; + protected onLabelShape: Text; + + protected offPowerSymbolCircle: Path; + protected offPowerSymbolLine: Path; + protected onPowerSymbolCircle: Path; + protected onPowerSymbolLine: Path; + protected constructor(protected widgetContext: WidgetContext, protected svgShape: Svg, + protected iconRegistry: MatIconRegistry, protected settings: PowerButtonWidgetSettings, protected value: boolean, protected disabled: boolean, protected onClick: () => void) { this.colors = createPowerButtonShapeColors(this.settings); + this.icons = {onButtonIcon: this.settings.onButtonIcon, offButtonIcon: this.settings.offButtonIcon}; this.onLabel = this.widgetContext.translate.instant('widgets.power-button.on-label').toUpperCase(); this.offLabel = this.widgetContext.translate.instant('widgets.power-button.off-label').toUpperCase(); this._drawShape(); } + public createIconElement(icon: string, size: number): Observable { + const isSvg = isSvgIcon(icon); + if (isSvg) { + const [namespace, iconName] = splitIconName(icon); + return this.iconRegistry + .getNamedSvgIcon(iconName, namespace) + .pipe( + take(1), + map((svgElement) => { + const element = new Element(svgElement.firstChild); + const box = element.bbox(); + const scale = size / box.height; + element.scale(scale); + return element; + }), + catchError(() => of(null) + )); + } else { + const iconName = splitIconName(icon)[1]; + const textElement = this.svgShape.text(iconName); + const fontSetClasses = ( + this.iconRegistry.getDefaultFontSetClass() + ).filter(className => className.length > 0); + fontSetClasses.forEach(className => textElement.addClass(className)); + textElement.font({size: `${size}px`}); + textElement.attr({ + style: `font-size: ${size}px`, + 'text-anchor': 'start' + }); + const tspan = textElement.first(); + tspan.attr({ + 'dominant-baseline': 'hanging' + }); + return of(textElement); + } + } + public setValue(value: boolean) { if (this.value !== value) { this.value = value; @@ -330,6 +412,106 @@ export abstract class PowerButtonShape { } } + public drawOffShape(centerGroup: G, label: boolean, labelWeight?: string, circleStroke?: boolean) { + if (this.icons.offButtonIcon.showIcon) { + this.createIconElement(this.icons.offButtonIcon.icon, this.icons.offButtonIcon.iconSize).subscribe(icon => + this.offPowerSymbolIcon = icon.center(cx, cy).addTo(centerGroup)); + } else { + if (label) { + this.offLabelShape = this.createOffLabel(labelWeight).addTo(centerGroup); + } else { + this.offPowerSymbolCircle = this.svgShape.path(circleStroke ? powerCircle : powerCircleStroke).center(cx, cy).addTo(centerGroup); + this.offPowerSymbolLine = this.svgShape.path(circleStroke ? powerLine : powerLineStroke).center(cx, cy-12).addTo(centerGroup); + } + } + } + + public drawOnShape(onCenterGroup?: G, label?: boolean, labelWeight?: string, circleStroke?: boolean, mask?: Circle) { + if (this.icons.onButtonIcon.showIcon) { + this.createIconElement(this.icons.onButtonIcon.icon, this.icons.onButtonIcon.iconSize).subscribe(icon => { + this.onPowerSymbolIcon = icon.center(cx, cy); + if (isDefinedAndNotNull(onCenterGroup)) { + this.onPowerSymbolIcon.addTo(onCenterGroup); + } + if (isDefinedAndNotNull(mask)) { + this.createMask(mask, [this.onPowerSymbolIcon]); + } + }); + } else { + if (label) { + this.onLabelShape = this.createOnLabel(labelWeight); + if (isDefinedAndNotNull(onCenterGroup)) { + this.onLabelShape.addTo(onCenterGroup); + } + if (isDefinedAndNotNull(mask)) { + this.createMask(mask, [this.onLabelShape]); + } + } else { + this.onPowerSymbolCircle = this.svgShape.path(circleStroke ? powerCircle : powerCircleStroke).center(cx, cy); + this.onPowerSymbolLine = this.svgShape.path(circleStroke ? powerLine : powerLineStroke).center(cx, cy-12); + if (isDefinedAndNotNull(onCenterGroup)) { + this.onPowerSymbolCircle.addTo(onCenterGroup); + this.onPowerSymbolLine.addTo(onCenterGroup); + } + if (isDefinedAndNotNull(mask)) { + this.createMask(mask, [this.onPowerSymbolCircle, this.onPowerSymbolLine]); + } + } + } + } + + public onCenterTimeLine(timeline: Timeline, label: boolean) { + if (this.icons.onButtonIcon.showIcon) { + this.onPowerSymbolIcon.timeline(timeline); + } else { + if (label) { + this.onLabelShape.timeline(timeline); + } else { + this.onPowerSymbolCircle.timeline(timeline); + this.onPowerSymbolLine.timeline(timeline); + } + } + } + + public offCenterColor(mainColor: PowerButtonColor, label: boolean) { + if (this.icons.offButtonIcon.showIcon) { + this.offPowerSymbolIcon.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + } else { + if (label) { + this.offLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + } else { + this.offPowerSymbolCircle.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offPowerSymbolLine.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + } + } + } + + public onCenterColor(mainColor: PowerButtonColor, label: boolean) { + if (this.icons.onButtonIcon.showIcon) { + this.onPowerSymbolIcon.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + } else { + if (label) { + this.onLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + } else { + this.onPowerSymbolCircle.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.onPowerSymbolLine.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + } + } + } + + public buttonAnimation(scale: number, label: boolean) { + if (this.icons.onButtonIcon.showIcon) { + powerButtonAnimation(this.onPowerSymbolIcon).transform({scale}); + } else { + if (label) { + powerButtonAnimation(this.onLabelShape).transform({scale, origin: {x: cx, y: cy}}); + } else { + powerButtonAnimation(this.onPowerSymbolCircle).transform({scale, origin: {x: cx, y: cy}}); + powerButtonAnimation(this.onPowerSymbolLine).transform({scale, origin: {x: cx, y: cy}}); + } + } + } + private _drawShape() { this.backgroundShape = this.svgShape.circle(powerButtonShapeSize).center(cx, cy) @@ -530,9 +712,7 @@ class DefaultPowerButtonShape extends PowerButtonShape { private outerBorder: Circle; private outerBorderMask: Circle; - private offLabelShape: Text; private onCircleShape: Circle; - private onLabelShape: Text; private pressedShadow: InnerShadowCircle; private pressedTimeline: Timeline; private centerGroup: G; @@ -543,22 +723,20 @@ class DefaultPowerButtonShape extends PowerButtonShape { this.outerBorderMask = this.svgShape.circle(powerButtonShapeSize - 20).center(cx, cy); this.createMask(this.outerBorder, [this.outerBorderMask]); this.centerGroup = this.svgShape.group(); - this.offLabelShape = this.createOffLabel().addTo(this.centerGroup); - this.onCircleShape = this.svgShape.circle(powerButtonShapeSize - 20) - .center(cx, cy); - this.onLabelShape = this.createOnLabel(); - this.createMask(this.onCircleShape, [this.onLabelShape]); + this.drawOffShape(this.centerGroup, true); + this.onCircleShape = this.svgShape.circle(powerButtonShapeSize - 20).center(cx, cy); + this.drawOnShape(null, true, '', false, this.onCircleShape); this.pressedShadow = new InnerShadowCircle(this.svgShape, powerButtonShapeSize - 20, cx, cy, 0, 0); this.pressedTimeline = new Timeline(); this.centerGroup.timeline(this.pressedTimeline); - this.onLabelShape.timeline(this.pressedTimeline); + this.onCenterTimeLine(this.pressedTimeline, true); this.pressedShadow.timeline(this.pressedTimeline); } protected drawColorState(mainColor: PowerButtonColor) { this.outerBorder.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); - this.offLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offCenterColor(mainColor, true); this.onCircleShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); } @@ -578,14 +756,14 @@ class DefaultPowerButtonShape extends PowerButtonShape { this.pressedTimeline.finish(); const pressedScale = 0.75; powerButtonAnimation(this.centerGroup).transform({scale: pressedScale}); - powerButtonAnimation(this.onLabelShape).transform({scale: pressedScale, origin: {x: cx, y: cy}}); + this.buttonAnimation(pressedScale, true); this.pressedShadow.animate(6, 0.6); } protected onPressEnd() { this.pressedTimeline.finish(); powerButtonAnimation(this.centerGroup).transform({scale: 1}); - powerButtonAnimation(this.onLabelShape).transform({scale: 1, origin: {x: cx, y: cy}}); + this.buttonAnimation(1, true); this.pressedShadow.animateRestore(); } @@ -596,8 +774,6 @@ class SimplifiedPowerButtonShape extends PowerButtonShape { private outerBorder: Circle; private outerBorderMask: Circle; private onCircleShape: Circle; - private offLabelShape: Text; - private onLabelShape: Text; private pressedShadow: InnerShadowCircle; private pressedTimeline: Timeline; private centerGroup: G; @@ -608,22 +784,21 @@ class SimplifiedPowerButtonShape extends PowerButtonShape { this.outerBorderMask = this.svgShape.circle(powerButtonShapeSize - 4).center(cx, cy); this.createMask(this.outerBorder, [this.outerBorderMask]); this.centerGroup = this.svgShape.group(); - this.offLabelShape = this.createOffLabel().addTo(this.centerGroup); + this.drawOffShape(this.centerGroup, true); this.onCircleShape = this.svgShape.circle(powerButtonShapeSize).center(cx, cy); - this.onLabelShape = this.createOnLabel(); - this.createMask(this.onCircleShape, [this.onLabelShape]); + this.drawOnShape(null, true, '', false, this.onCircleShape) this.pressedShadow = new InnerShadowCircle(this.svgShape, powerButtonShapeSize - 4, cx, cy, 0, 0); this.pressedTimeline = new Timeline(); this.centerGroup.timeline(this.pressedTimeline); - this.onLabelShape.timeline(this.pressedTimeline); + this.onCenterTimeLine(this.pressedTimeline, true); this.pressedShadow.timeline(this.pressedTimeline); } protected drawColorState(mainColor: PowerButtonColor) { this.outerBorder.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); this.onCircleShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); - this.offLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offCenterColor(mainColor, true); } protected drawOff() { @@ -642,14 +817,14 @@ class SimplifiedPowerButtonShape extends PowerButtonShape { this.pressedTimeline.finish(); const pressedScale = 0.75; powerButtonAnimation(this.centerGroup).transform({scale: pressedScale}); - powerButtonAnimation(this.onLabelShape).transform({scale: pressedScale, origin: {x: cx, y: cy}}); + this.buttonAnimation(pressedScale, true); this.pressedShadow.animate(6, 0.6); } protected onPressEnd() { this.pressedTimeline.finish(); powerButtonAnimation(this.centerGroup).transform({scale: 1}); - powerButtonAnimation(this.onLabelShape).transform({scale: 1, origin: {x: cx, y: cy}}); + this.buttonAnimation(1, true); this.pressedShadow.animateRestore(); } } @@ -659,9 +834,7 @@ class OutlinedPowerButtonShape extends PowerButtonShape { private outerBorderMask: Circle; private innerBorder: Circle; private innerBorderMask: Circle; - private offLabelShape: Text; private onCircleShape: Circle; - private onLabelShape: Text; private pressedShadow: InnerShadowCircle; private pressedTimeline: Timeline; private centerGroup: G; @@ -677,25 +850,23 @@ class OutlinedPowerButtonShape extends PowerButtonShape { this.innerBorderMask = this.svgShape.circle(powerButtonShapeSize - 24).center(cx, cy); this.createMask(this.innerBorder, [this.innerBorderMask]); this.centerGroup = this.svgShape.group(); - this.offLabelShape = this.createOffLabel().addTo(this.centerGroup); + this.drawOffShape(this.centerGroup, true); this.onCenterGroup = this.svgShape.group(); - this.onCircleShape = this.svgShape.circle(powerButtonShapeSize - 28).center(cx, cy) - .addTo(this.onCenterGroup); - this.onLabelShape = this.createOnLabel(); - this.createMask(this.onCircleShape, [this.onLabelShape]); + this.onCircleShape = this.svgShape.circle(powerButtonShapeSize - 28).center(cx, cy).addTo(this.onCenterGroup); + this.drawOnShape(null, true, '', false, this.onCircleShape) this.pressedShadow = new InnerShadowCircle(this.svgShape, powerButtonShapeSize - 24, cx, cy, 0, 0); this.pressedTimeline = new Timeline(); this.centerGroup.timeline(this.pressedTimeline); this.onCenterGroup.timeline(this.pressedTimeline); - this.onLabelShape.timeline(this.pressedTimeline); + this.onCenterTimeLine(this.pressedTimeline, true); this.pressedShadow.timeline(this.pressedTimeline); } protected drawColorState(mainColor: PowerButtonColor) { this.outerBorder.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); this.innerBorder.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); - this.offLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offCenterColor(mainColor, true); this.onCircleShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); } @@ -714,7 +885,7 @@ class OutlinedPowerButtonShape extends PowerButtonShape { const pressedScale = 0.75; powerButtonAnimation(this.centerGroup).transform({scale: pressedScale}); powerButtonAnimation(this.onCenterGroup).transform({scale: 0.98}); - powerButtonAnimation(this.onLabelShape).transform({scale: pressedScale / 0.98, origin: {x: cx, y: cy}}); + this.buttonAnimation(pressedScale / 0.98, true); this.pressedShadow.animate(6, 0.6); } @@ -722,7 +893,7 @@ class OutlinedPowerButtonShape extends PowerButtonShape { this.pressedTimeline.finish(); powerButtonAnimation(this.centerGroup).transform({scale: 1}); powerButtonAnimation(this.onCenterGroup).transform({scale: 1}); - powerButtonAnimation(this.onLabelShape).transform({scale: 1, origin: {x: cx, y: cy}}); + this.buttonAnimation(1, true); this.pressedShadow.animateRestore(); } } @@ -737,35 +908,29 @@ class DefaultVolumePowerButtonShape extends PowerButtonShape { private innerShadow: InnerShadowCircle; //private innerShadowGradient: Gradient; //private innerShadowGradientStop: Stop; - private offLabelShape: Text; - private onCircleShape: Circle; - private onLabelShape: Text; + protected onCircleShape: Circle; private pressedTimeline: Timeline; private centerGroup: G; protected drawOffCenter(centerGroup: G) { - this.offLabelShape = this.createOffLabel('400').addTo(centerGroup); + this.drawOffShape(centerGroup, true, '400'); } protected drawOnCenter() { - this.onLabelShape = this.createOnLabel('400'); - } - - protected addOnCenterToMask(onCircleShape: Circle) { - this.createMask(onCircleShape,[this.onLabelShape]); + this.drawOnShape(null, true, '400', false, this.onCircleShape); } protected addOnCenterTimeLine(pressedTimeline: Timeline) { - this.onLabelShape.timeline(pressedTimeline); + this.onCenterTimeLine(pressedTimeline, true); } protected drawOffCenterColor(mainColor: PowerButtonColor) { - this.offLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offCenterColor(mainColor, true); } protected onCenterAnimation(scale: number) { - powerButtonAnimation(this.onLabelShape).transform({scale, origin: {x: cx, y: cy}}); + this.buttonAnimation(scale, true); } protected drawShape() { @@ -789,7 +954,6 @@ class DefaultVolumePowerButtonShape extends PowerButtonShape { this.drawOffCenter(this.centerGroup); this.onCircleShape = this.svgShape.circle(powerButtonShapeSize - 24).center(cx, cy); this.drawOnCenter(); - this.addOnCenterToMask(this.onCircleShape); this.innerShadow = new InnerShadowCircle(this.svgShape, powerButtonShapeSize - 24, cx, cy, 3, 0.3); this.pressedTimeline = new Timeline(); @@ -857,38 +1021,25 @@ class DefaultVolumePowerButtonShape extends PowerButtonShape { } class DefaultIconPowerButtonShape extends DefaultVolumePowerButtonShape { - private offPowerSymbolCircle: Path; - private offPowerSymbolLine: Path; - private onPowerSymbolCircle: Path; - private onPowerSymbolLine: Path; protected drawOffCenter(centerGroup: G) { - this.offPowerSymbolCircle = this.svgShape.path(powerCircle).center(cx, cy).addTo(centerGroup); - this.offPowerSymbolLine = this.svgShape.path(powerLine).center(cx, cy-12).addTo(centerGroup); + this.drawOffShape(centerGroup, false); } protected drawOnCenter() { - this.onPowerSymbolCircle = this.svgShape.path(powerCircle).center(cx, cy); - this.onPowerSymbolLine = this.svgShape.path(powerLine).center(cx, cy-12); - } - - protected addOnCenterToMask(onCircleShape: Circle) { - this.createMask(onCircleShape, [this.onPowerSymbolCircle, this.onPowerSymbolLine]); + this.drawOnShape(null, false, '', false, this.onCircleShape); } protected addOnCenterTimeLine(pressedTimeline: Timeline) { - this.onPowerSymbolCircle.timeline(pressedTimeline); - this.onPowerSymbolLine.timeline(pressedTimeline); + this.onCenterTimeLine(pressedTimeline, false); } protected drawOffCenterColor(mainColor: PowerButtonColor) { - this.offPowerSymbolCircle.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); - this.offPowerSymbolLine.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offCenterColor(mainColor, false); } protected onCenterAnimation(scale: number) { - powerButtonAnimation(this.onPowerSymbolCircle).transform({scale, origin: {x: cx, y: cy}}); - powerButtonAnimation(this.onPowerSymbolLine).transform({scale, origin: {x: cx, y: cy}}); + this.buttonAnimation(scale, false); } } @@ -896,8 +1047,6 @@ class SimplifiedVolumePowerButtonShape extends PowerButtonShape { private outerBorder: Circle; private outerBorderMask: Circle; - private offLabelShape: Text; - private onLabelShape: Text; private innerShadow: InnerShadowCircle; private pressedShadow: InnerShadowCircle; private pressedTimeline: Timeline; @@ -905,8 +1054,8 @@ class SimplifiedVolumePowerButtonShape extends PowerButtonShape { private onCenterGroup: G; protected drawCenterGroup(centerGroup: G, onCenterGroup: G) { - this.offLabelShape = this.createOffLabel().addTo(centerGroup); - this.onLabelShape = this.createOnLabel().addTo(onCenterGroup); + this.drawOffShape(centerGroup, true); + this.drawOnShape(onCenterGroup, true); } protected drawShape() { @@ -926,8 +1075,8 @@ class SimplifiedVolumePowerButtonShape extends PowerButtonShape { } protected drawColorState(mainColor: PowerButtonColor){ - this.offLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); - this.onLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offCenterColor(mainColor, true); + this.onCenterColor(mainColor, true); } protected drawOff() { @@ -970,23 +1119,15 @@ class SimplifiedVolumePowerButtonShape extends PowerButtonShape { } class SimplifiedIconPowerButtonShape extends SimplifiedVolumePowerButtonShape { - private offPowerSymbolCircle: Path; - private offPowerSymbolLine: Path; - private onPowerSymbolCircle: Path; - private onPowerSymbolLine: Path; protected drawCenterGroup(centerGroup: G, onCenterGroup: G) { - this.offPowerSymbolCircle = this.svgShape.path(powerCircle).center(cx, cy).addTo(centerGroup); - this.offPowerSymbolLine = this.svgShape.path(powerLine).center(cx, cy-12).addTo(centerGroup); - this.onPowerSymbolCircle = this.svgShape.path(powerCircle).center(cx, cy).addTo(onCenterGroup); - this.onPowerSymbolLine = this.svgShape.path(powerLine).center(cx, cy-12).addTo(onCenterGroup); + this.drawOffShape(centerGroup, false); + this.drawOnShape(onCenterGroup, false, '', false); } protected drawColorState(mainColor: PowerButtonColor) { - this.offPowerSymbolCircle.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); - this.offPowerSymbolLine.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); - this.onPowerSymbolCircle.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); - this.onPowerSymbolLine.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offCenterColor(mainColor, false); + this.onCenterColor(mainColor, false); } } @@ -996,36 +1137,30 @@ class OutlinedVolumePowerButtonShape extends PowerButtonShape { private outerBorderGradient: Gradient; private innerBorder: Circle; private innerBorderMask: Circle; - private offLabelShape: Text; - private onCircleShape: Circle; - private onLabelShape: Text; + protected onCircleShape: Circle; private pressedShadow: InnerShadowCircle; private pressedTimeline: Timeline; private centerGroup: G; private onCenterGroup: G; protected drawOffCenter(centerGroup: G) { - this.offLabelShape = this.createOffLabel('800').addTo(centerGroup); + this.drawOffShape(centerGroup, true, '800'); } protected drawOnCenter() { - this.onLabelShape = this.createOnLabel('800'); - } - - protected addOnCenterToMask(onCircleShape: Circle) { - this.createMask(onCircleShape,[this.onLabelShape]); + this.drawOnShape(null, true, '800', false, this.onCircleShape); } protected addOnCenterTimeLine(pressedTimeline: Timeline) { - this.onLabelShape.timeline(pressedTimeline); + this.onCenterTimeLine(pressedTimeline, true); } protected drawOffCenterColor(mainColor: PowerButtonColor) { - this.offLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offCenterColor(mainColor, true); } protected onCenterAnimation(scale: number) { - powerButtonAnimation(this.onLabelShape).transform({scale, origin: {x: cx, y: cy}}); + this.buttonAnimation(scale, true); } protected drawShape() { @@ -1047,7 +1182,6 @@ class OutlinedVolumePowerButtonShape extends PowerButtonShape { this.onCircleShape = this.svgShape.circle(powerButtonShapeSize - 30).center(cx, cy) .addTo(this.onCenterGroup); this.drawOnCenter(); - this.addOnCenterToMask(this.onCircleShape); this.pressedShadow = new InnerShadowCircle(this.svgShape, powerButtonShapeSize - 30, cx, cy, 0, 0); this.backgroundShape.addClass('tb-small-shadow'); @@ -1102,37 +1236,24 @@ class OutlinedVolumePowerButtonShape extends PowerButtonShape { } class OutlinedIconPowerButtonShape extends OutlinedVolumePowerButtonShape { - private offPowerSymbolCircle: Path; - private offPowerSymbolLine: Path; - private onPowerSymbolCircle: Path; - private onPowerSymbolLine: Path; protected drawOffCenter(centerGroup: G) { - this.offPowerSymbolCircle = this.svgShape.path(powerCircleStroke).center(cx, cy).addTo(centerGroup); - this.offPowerSymbolLine = this.svgShape.path(powerLineStroke).center(cx, cy-12).addTo(centerGroup); + this.drawOffShape(centerGroup, false, '', true); } protected drawOnCenter() { - this.onPowerSymbolCircle = this.svgShape.path(powerCircleStroke).center(cx, cy); - this.onPowerSymbolLine = this.svgShape.path(powerLineStroke).center(cx, cy-12); - } - - protected addOnCenterToMask(onCircleShape: Circle) { - this.createMask(onCircleShape, [this.onPowerSymbolCircle, this.onPowerSymbolLine]); + this.drawOnShape(null, false, '', true, this.onCircleShape); } protected addOnCenterTimeLine(pressedTimeline: Timeline) { - this.onPowerSymbolCircle.timeline(pressedTimeline); - this.onPowerSymbolLine.timeline(pressedTimeline); + this.onCenterTimeLine(pressedTimeline, false); } protected drawOffCenterColor(mainColor: PowerButtonColor) { - this.offPowerSymbolCircle.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); - this.offPowerSymbolLine.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offCenterColor(mainColor, false); } protected onCenterAnimation(scale: number) { - powerButtonAnimation(this.onPowerSymbolCircle).transform({scale, origin: {x: cx, y: cy}}); - powerButtonAnimation(this.onPowerSymbolLine).transform({scale, origin: {x: cx, y: cy}}); + this.buttonAnimation(scale, false); } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.html new file mode 100644 index 0000000000..6b28d44066 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.html @@ -0,0 +1,43 @@ + +
        +
        + +
        +
        +
        +
        +
        {{ valueText }}
        +
        +
        +
        +
        + +
        diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.scss b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.scss new file mode 100644 index 0000000000..47f15e9dd5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.scss @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.tb-value-stepper-panel { + width: 100%; + height: 100%; + position: relative; + display: flex; + flex-direction: column; + gap: 16px; + padding: 20px 24px 24px 24px; + > div:not(.tb-value-stepper-overlay) { + z-index: 1; + } + .tb-value-stepper-overlay { + position: absolute; + top: 12px; + left: 12px; + bottom: 12px; + right: 12px; + } + div.tb-widget-title { + padding: 0; + } + .tb-value-stepper-content { + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; + min-width: 0; + min-height: 0; + height: 100%; + .tb-value-stepper-value-box { + display: flex; + align-items: center; + justify-content: center; + flex: 1; + height: 32px; + padding: 0 12px; + border-radius: 4px; + .tb-value-stepper-value-disabled { + color: rgba(0, 0, 0, 0.38) !important; + } + &-disabled { + border-color: rgba(0, 0, 0, 0.38); + } + } + .tb-button-shape { + width: 32px; + height: 32px; + display: flex; + align-items: center; + justify-content: center; + &-left { + margin-right: 12px; + } + &-right { + margin-left: 12px; + } + svg { + .tb-small-shadow { + filter: drop-shadow(0px 0px 4px rgba(0, 0, 0, 0.2)); + } + .tb-shadow { + filter: drop-shadow(0px 0px 10px rgba(0, 0, 0, 0.15)); + } + } + &.tb-button-pointer { + svg { + .tb-hover-circle { + cursor: pointer; + } + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.ts new file mode 100644 index 0000000000..d419aef0d4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.component.ts @@ -0,0 +1,388 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + ElementRef, + NgZone, + OnDestroy, + OnInit, + Renderer2, + ViewChild, + ViewEncapsulation +} from '@angular/core'; +import { BasicActionWidgetComponent, ValueSetter } from '@home/components/widget/lib/action/action-widget.models'; +import { backgroundStyle, ComponentStyle, overlayStyle, textStyle } from '@shared/models/widget-settings.models'; +import { BehaviorSubject, combineLatest, Observable, Subject } from 'rxjs'; +import { ImagePipe } from '@shared/pipe/image.pipe'; +import { DomSanitizer } from '@angular/platform-browser'; +import { ValueType } from '@shared/models/constants'; +import { + PowerButtonLayout, + PowerButtonShape, + powerButtonShapeSize, + PowerButtonWidgetSettings +} from '@home/components/widget/lib/rpc/power-button-widget.models'; +import { SVG, Svg } from '@svgdotjs/svg.js'; +import { MatIconRegistry } from '@angular/material/icon'; +import { formatValue, isDefinedAndNotNull, isNumeric } from '@core/utils'; +import { + valueStepperDefaultSettings, + ValueStepperWidgetSettings +} from '@home/components/widget/lib/rpc/value-stepper-widget.models'; +import { UtilsService } from '@core/services/utils.service'; +import { takeUntil } from 'rxjs/operators'; + +@Component({ + selector: 'tb-value-stepper-widget', + templateUrl: './value-stepper-widget.component.html', + styleUrls: ['../action/action-widget.scss', './value-stepper-widget.component.scss'], + encapsulation: ViewEncapsulation.None +}) +export class ValueStepperWidgetComponent extends + BasicActionWidgetComponent implements OnInit, AfterViewInit, OnDestroy { + + @ViewChild('leftButton', {static: false}) + leftButton: ElementRef; + + @ViewChild('rightButton', {static: false}) + rightButton: ElementRef; + + @ViewChild('stepperContent', {static: false}) + stepperContent: ElementRef; + + @ViewChild('valueBoxContainer', {static: false}) + valueBox: ElementRef; + + @ViewChild('value', {static: false}) + valueElement: ElementRef; + + settings: ValueStepperWidgetSettings; + + backgroundStyle$: Observable; + overlayStyle: ComponentStyle = {}; + padding: string; + + valueStyle: ComponentStyle = {}; + value: number = null; + + autoScale = false; + + showValueBox = true; + showLeftButton = true; + showRightButton = true; + + valueText = 'N/A'; + + disabledState$ = new BehaviorSubject(false); + + private prevValue: number = null; + private shapeResize$: ResizeObserver; + private drawSvgShapePending = false; + private svgShapeLeft: Svg; + private svgShapeRight: Svg; + private powerButtonSvgShapeLeft: PowerButtonShape; + private powerButtonSvgShapeRight: PowerButtonShape; + + private disabledState = false; + public leftDisabledState = false; + public rightDisabledState = false; + + private valueSetterLeft: ValueSetter; + private valueSetterRight: ValueSetter; + + private leftDisabledState$ = new BehaviorSubject(false); + private rightDisabledState$ = new BehaviorSubject(false); + private readonly destroy$ = new Subject(); + + constructor(protected imagePipe: ImagePipe, + protected sanitizer: DomSanitizer, + private renderer: Renderer2, + private iconRegistry: MatIconRegistry, + private utils: UtilsService, + private elementRef: ElementRef, + protected cd: ChangeDetectorRef, + protected zone: NgZone) { + super(cd); + } + + ngOnInit(): void { + super.ngOnInit(); + this.settings = {...valueStepperDefaultSettings, ...this.ctx.settings}; + + this.autoScale = this.settings.appearance.autoScale; + + this.backgroundStyle$ = backgroundStyle(this.settings.background, this.imagePipe, this.sanitizer); + this.overlayStyle = overlayStyle(this.settings.background.overlay); + this.padding = this.settings.background.overlay.enabled ? undefined : this.settings.padding; + + this.showValueBox = this.settings.appearance.showValueBox; + this.showLeftButton = this.settings.buttonAppearance.leftButton.showButton; + this.showRightButton = this.settings.buttonAppearance.rightButton.showButton; + this.valueStyle = textStyle(this.settings.appearance.valueFont); + this.valueStyle.color = this.settings.appearance.valueColor; + + if (this.showValueBox) { + const valueBoxCss = `.tb-value-stepper-value-box {\n`+ + `border: ${this.settings.appearance.showBorder ? + `${this.settings.appearance.borderWidth}px solid ${this.settings.appearance.borderColor}` : + 'none'};\n`+ + `background-color: ${this.settings.appearance.valueBoxBackground}` + + `}`; + this.utils.applyCssToElement(this.renderer, this.elementRef.nativeElement, 'tb-value-stepper-value-box', valueBoxCss); + } + + const getInitialStateSettings = + {...this.settings.initialState, actionLabel: this.ctx.translate.instant('widgets.slider.initial-value')}; + this.createValueGetter(getInitialStateSettings, ValueType.INTEGER, { + next: (value) => this.onValue(value) + }); + const disabledStateSettings = + {...this.settings.disabledState, actionLabel: this.ctx.translate.instant('widgets.rpc-state.disabled-state')}; + this.createValueGetter(disabledStateSettings, ValueType.BOOLEAN, { + next: (value) => this.disabledState$.next(value) + }); + + const leftButtonClick = {...this.settings.leftButtonClick, + actionLabel: this.ctx.translate.instant('widgets.slider.on-value-change')}; + this.valueSetterLeft = this.createValueSetter(leftButtonClick); + + const rightButtonClick = {...this.settings.rightButtonClick, + actionLabel: this.ctx.translate.instant('widgets.slider.on-value-change')}; + this.valueSetterRight = this.createValueSetter(rightButtonClick); + + combineLatest([ + this.loading$, + this.disabledState$.asObservable(), + this.leftDisabledState$.asObservable() + ]).pipe( + takeUntil(this.destroy$) + ).subscribe(value => { + const state = value.includes(true); + this.updateLeftDisabledState(state) + }); + + combineLatest([ + this.loading$, + this.disabledState$.asObservable(), + this.rightDisabledState$.asObservable() + ]).pipe( + takeUntil(this.destroy$) + ).subscribe(value => { + const state = value.includes(true); + this.updateRightDisabledState(state) + }); + } + + ngAfterViewInit(): void { + if (this.drawSvgShapePending) { + this.drawSvg(); + } + super.ngAfterViewInit(); + } + + ngOnDestroy() { + if (this.shapeResize$) { + this.shapeResize$.disconnect(); + } + this.destroy$.next(); + this.destroy$.complete(); + super.ngOnDestroy(); + } + + public onInit() { + super.onInit(); + const borderRadius = this.ctx.$widgetElement.css('borderRadius'); + this.overlayStyle = {...this.overlayStyle, ...{borderRadius}}; + if (this.leftButton || this.rightButton) { + this.drawSvg(); + } else { + this.drawSvgShapePending = true; + } + this.cd.detectChanges(); + } + + private onValue(value: number): void { + this.value = value; + this.prevValue = value; + if ((this.value + this.settings.appearance.valueStep) >= this.settings.appearance.maxValueRange) { + this.rightDisabledState$.next(true); + } else { + this.rightDisabledState$.next(false); + } + if ((this.value - this.settings.appearance.valueStep) <= this.settings.appearance.minValueRange) { + this.leftDisabledState$.next(true); + } else { + this.leftDisabledState$.next(false); + } + this.updateValueText(); + this.cd.markForCheck(); + } + + private updateValueText() { + if (isDefinedAndNotNull(this.value) && isNumeric(this.value)) { + this.valueText = formatValue(this.value, this.settings.appearance.valueDecimals, this.settings.appearance.valueUnits, false); + } else { + this.valueText = 'N/A'; + } + } + + private onClick(rightButtonClick: boolean = false) { + this.updateValueText(); + if (!this.ctx.isEdit && !this.ctx.isPreview && !this.disabledState) { + const prevValue = this.prevValue; + const targetValue = rightButtonClick ? + (this.value + this.settings.appearance.valueStep) : + (this.value - this.settings.appearance.valueStep); + this.updateValue(rightButtonClick ? this.valueSetterRight : this.valueSetterLeft, targetValue, { + next: () => this.onValue(targetValue), + error: () => this.onValue(prevValue) + }); + } + } + + private drawSvg() { + let leftButtonSetting: PowerButtonWidgetSettings; + let rightButtonSetting: PowerButtonWidgetSettings; + if (this.showLeftButton) { + this.svgShapeLeft = SVG().addTo(this.leftButton.nativeElement).size(powerButtonShapeSize, powerButtonShapeSize); + this.renderer.setStyle(this.svgShapeLeft.node, 'overflow', 'visible'); + this.renderer.setStyle(this.svgShapeLeft.node, 'user-select', 'none'); + leftButtonSetting = { + layout: PowerButtonLayout[this.settings.appearance.type], + onButtonIcon: { + showIcon: true, + icon: this.settings.buttonAppearance.leftButton.icon, + iconSize: this.settings.buttonAppearance.leftButton.iconSize * 1.7, + iconSizeUnit: this.settings.buttonAppearance.leftButton.iconSizeUnit + }, + offButtonIcon: { + showIcon: true, + icon: this.settings.buttonAppearance.leftButton.icon, + iconSize: this.settings.buttonAppearance.leftButton.iconSize * 1.7, + iconSizeUnit: this.settings.buttonAppearance.leftButton.iconSizeUnit + }, + mainColorOn: this.settings.buttonAppearance.leftButton.mainColorOn, + backgroundColorOn: this.settings.buttonAppearance.leftButton.backgroundColorOn, + mainColorOff: this.settings.buttonAppearance.leftButton.mainColorOff, + backgroundColorOff: this.settings.buttonAppearance.leftButton.backgroundColorOff, + mainColorDisabled: this.settings.buttonAppearance.leftButton.mainColorDisabled, + backgroundColorDisabled: this.settings.buttonAppearance.leftButton.backgroundColorDisabled + }; + } + if (this.showRightButton) { + this.svgShapeRight = SVG().addTo(this.rightButton.nativeElement).size(powerButtonShapeSize, powerButtonShapeSize); + this.renderer.setStyle(this.svgShapeRight.node, 'overflow', 'visible'); + this.renderer.setStyle(this.svgShapeRight.node, 'user-select', 'none'); + + rightButtonSetting = { + layout: PowerButtonLayout[this.settings.appearance.type], + onButtonIcon: { + showIcon: true, + icon: this.settings.buttonAppearance.rightButton.icon, + iconSize: this.settings.buttonAppearance.rightButton.iconSize * 1.7, + iconSizeUnit: this.settings.buttonAppearance.rightButton.iconSizeUnit + }, + offButtonIcon: { + showIcon: true, + icon: this.settings.buttonAppearance.rightButton.icon, + iconSize: this.settings.buttonAppearance.rightButton.iconSize * 1.7, + iconSizeUnit: this.settings.buttonAppearance.rightButton.iconSizeUnit + }, + mainColorOn: this.settings.buttonAppearance.rightButton.mainColorOn, + backgroundColorOn: this.settings.buttonAppearance.rightButton.backgroundColorOn, + mainColorOff: this.settings.buttonAppearance.rightButton.mainColorOff, + backgroundColorOff: this.settings.buttonAppearance.rightButton.backgroundColorOff, + mainColorDisabled: this.settings.buttonAppearance.rightButton.mainColorDisabled, + backgroundColorDisabled: this.settings.buttonAppearance.rightButton.backgroundColorDisabled + }; + } + + this.zone.run(() => { + if (this.showLeftButton) { + this.powerButtonSvgShapeLeft = PowerButtonShape.fromSettings(this.ctx, this.svgShapeLeft, this.iconRegistry, + leftButtonSetting , true, this.leftDisabledState, () => this.onClick()); + } + if (this.showRightButton) { + this.powerButtonSvgShapeRight = PowerButtonShape.fromSettings(this.ctx, this.svgShapeRight, this.iconRegistry, + rightButtonSetting, true, this.rightDisabledState, () => this.onClick(true)); + } + }); + + this.shapeResize$ = new ResizeObserver(() => { + this.onResize(); + }); + if (this.autoScale) { + this.shapeResize$.observe(this.stepperContent.nativeElement); + } + if (this.showLeftButton) { + this.shapeResize$.observe(this.leftButton.nativeElement); + } + if (this.showRightButton) { + this.shapeResize$.observe(this.rightButton.nativeElement); + } + this.onResize(); + } + + private updateLeftDisabledState(disabled: boolean) { + this.leftDisabledState = disabled; + this.powerButtonSvgShapeLeft?.setDisabled(this.leftDisabledState); + this.cd.markForCheck(); + } + + + private updateRightDisabledState(disabled: boolean) { + this.rightDisabledState = disabled; + this.powerButtonSvgShapeRight?.setDisabled(this.rightDisabledState); + this.cd.markForCheck(); + } + + private onResize() { + const panelWidth = this.stepperContent.nativeElement.getBoundingClientRect().width; + const panelHeight = this.stepperContent.nativeElement.getBoundingClientRect().height; + + const minAspect = 0.2; + const avgContentHeight = 32; + const targetHeight = panelWidth * Math.min(panelHeight / panelWidth, minAspect); + const multiplier = targetHeight / avgContentHeight; + const size = avgContentHeight * multiplier; + + if (this.showValueBox) { + this.renderer.setStyle(this.valueBox?.nativeElement, 'height', `${size}px`); + this.renderer.setStyle(this.valueElement?.nativeElement, 'font-size', `${this.settings.appearance.valueFont.size * multiplier}px`); + } + if (this.showLeftButton) { + this.renderer.setStyle(this.leftButton?.nativeElement, 'width', `${size}px`); + this.renderer.setStyle(this.leftButton?.nativeElement, 'height', `${size}px`); + } + if (this.showRightButton) { + this.renderer.setStyle(this.rightButton?.nativeElement, 'width', `${size}px`); + this.renderer.setStyle(this.rightButton?.nativeElement, 'height', `${size}px`); + } + if (size) { + const scale = size / powerButtonShapeSize; + if (this.showLeftButton) { + this.renderer.setStyle(this.svgShapeLeft?.node, 'transform', `scale(${scale})`); + } + if (this.showRightButton) { + this.renderer.setStyle(this.svgShapeRight?.node, 'transform', `scale(${scale})`); + } + } + } + +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.models.ts new file mode 100644 index 0000000000..02471bfc05 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/value-stepper-widget.models.ts @@ -0,0 +1,239 @@ +import { + DataToValueType, + GetValueAction, + GetValueSettings, SetValueAction, + SetValueSettings, ValueToDataType +} from '@shared/models/action-widget-settings.models'; +import { defaultWidgetAction, WidgetAction } from '@shared/models/widget.models'; +import { + ButtonToggleAppearance, segmentedButtonDefaultAppearance, + SegmentedButtonWidgetSettings +} from '@home/components/widget/lib/button/segmented-button-widget.models'; +import { WidgetButtonCustomStyles, WidgetButtonType } from '@shared/components/button/widget-button.models'; +import { BackgroundSettings, BackgroundType, cssUnit, Font } from '@shared/models/widget-settings.models'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; + + +const defaultMainColor = '#305680'; + +export enum ValueStepperType { + simplified = 'simplified', + default = 'default', + default_volume = 'default_volume' +} + +export const valueStepperTypes = Object.keys(ValueStepperType) as ValueStepperType[]; + +export const valueStepperTypeTranslations = new Map( + [ + [ValueStepperType.simplified, 'widgets.value-stepper.simplified'], + [ValueStepperType.default, 'widgets.value-stepper.filled'], + [ValueStepperType.default_volume, 'widgets.value-stepper.volume'] + ] +); + +export const valueStepperTypeImages = new Map( + [ + [ValueStepperType.simplified, 'assets/widget/value-stepper/simplified.svg'], + [ValueStepperType.default, 'assets/widget/value-stepper/filled.svg'], + [ValueStepperType.default_volume, 'assets/widget/value-stepper/volume.svg'] + ] +); + +export interface ValueStepperWidgetSettings { + initialState: GetValueSettings; + leftButtonClick: SetValueSettings; + rightButtonClick: SetValueSettings; + disabledState: GetValueSettings; + + appearance: ValueStepperAppearance; + buttonAppearance: { + leftButton: ValueStepperButtonAppearance; + rightButton: ValueStepperButtonAppearance; + } + + background: BackgroundSettings; + padding: string; +} + +export interface ValueStepperAppearance { + type: ValueStepperType; + autoScale: boolean; + minValueRange: number; + maxValueRange: number; + valueStep: number; + showValueBox: boolean; + valueUnits: string; + valueDecimals: number; + valueFont: Font; + valueColor: string; + valueBoxBackground: string; + showBorder: boolean; + borderWidth: number; + borderColor: string; +} + +export interface ValueStepperButtonAppearance { + showButton: boolean; + icon: string; + iconSize: number; + iconSizeUnit: cssUnit; + mainColorOn: string; + backgroundColorOn: string; + mainColorOff: string; + backgroundColorOff: string; + mainColorDisabled: string; + backgroundColorDisabled: string; + customStyle: WidgetButtonCustomStyles; +} + +export const valueStepperDefaultAppearance: ValueStepperAppearance = { + type: ValueStepperType.simplified, + autoScale: true, + minValueRange: -100, + maxValueRange: 100, + valueStep: 0.5, + showValueBox: true, + valueUnits: '', + valueDecimals: 1, + valueFont: { + family: 'Roboto', + weight: '500', + style: 'normal', + size: 16, + sizeUnit: 'px', + lineHeight: '24px' + }, + valueColor: '#000', + valueBoxBackground: 'rgba(0, 0, 0, 0.12)', + showBorder: true, + borderWidth: 1, + borderColor: defaultMainColor +} + +export const valueStepperButtonDefaultAppearance: ValueStepperButtonAppearance = { + showButton: true, + icon: '', + iconSize: 24, + iconSizeUnit: 'px', + + mainColorOn: '#3F52DD', + backgroundColorOn: '#FFFFFF', + mainColorOff: '#A2A2A2', + backgroundColorOff: '#FFFFFF', + mainColorDisabled: 'rgba(0,0,0,0.12)', + backgroundColorDisabled: '#FFFFFF', + customStyle: { + enabled: null, + hovered: null, + pressed: null, + activated: null, + disabled: null + } +} + +export const valueStepperDefaultSettings: ValueStepperWidgetSettings = { + initialState: { + action: GetValueAction.EXECUTE_RPC, + defaultValue: 0, + executeRpc: { + method: 'getState', + requestTimeout: 5000, + requestPersistent: false, + persistentPollingInterval: 1000 + }, + getAttribute: { + key: 'state', + scope: null + }, + getTimeSeries: { + key: 'state' + }, + getAlarmStatus: { + severityList: null, + typeList: null + }, + dataToValue: { + type: DataToValueType.NONE, + compareToValue: true, + dataToValueFunction: '/* Should return integer value */\nreturn data;' + } + }, + disabledState: { + action: GetValueAction.DO_NOTHING, + defaultValue: false, + getAttribute: { + key: 'state', + scope: null + }, + getTimeSeries: { + key: 'state' + }, + getAlarmStatus: { + severityList: null, + typeList: null + }, + dataToValue: { + type: DataToValueType.NONE, + compareToValue: true, + dataToValueFunction: '/* Should return boolean value */\nreturn data;' + } + }, + leftButtonClick: { + action: SetValueAction.EXECUTE_RPC, + executeRpc: { + method: 'setState', + requestTimeout: 5000, + requestPersistent: false, + persistentPollingInterval: 1000 + }, + setAttribute: { + key: 'state', + scope: AttributeScope.SERVER_SCOPE + }, + putTimeSeries: { + key: 'state' + }, + valueToData: { + type: ValueToDataType.VALUE, + constantValue: 0, + valueToDataFunction: '/* Convert input integer value to RPC parameters or attribute/time-series value */\nreturn value;' + } + }, + rightButtonClick: { + action: SetValueAction.EXECUTE_RPC, + executeRpc: { + method: 'setState', + requestTimeout: 5000, + requestPersistent: false, + persistentPollingInterval: 1000 + }, + setAttribute: { + key: 'state', + scope: AttributeScope.SERVER_SCOPE + }, + putTimeSeries: { + key: 'state' + }, + valueToData: { + type: ValueToDataType.VALUE, + constantValue: 0, + valueToDataFunction: '/* Convert input integer value to RPC parameters or attribute/time-series value */\nreturn value;' + } + }, + appearance: valueStepperDefaultAppearance, + buttonAppearance: { + leftButton: {...valueStepperButtonDefaultAppearance, icon: 'arrow_back_ios_new'}, + rightButton: {...valueStepperButtonDefaultAppearance, icon: 'arrow_forward_ios'} + }, + background: { + type: BackgroundType.color, + color: '#fff', + overlay: { + enabled: false, + color: 'rgba(255,255,255,0.72)', + blur: 3 + } + }, + padding: '12px' +}; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/value-stepper-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/value-stepper-widget-settings.component.html new file mode 100644 index 0000000000..ebc106794e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/value-stepper-widget-settings.component.html @@ -0,0 +1,266 @@ + + +
        +
        widgets.value-stepper.behavior
        +
        +
        widgets.value-stepper.initial-state
        + +
        +
        +
        widgets.value-stepper.left-button-click
        + +
        +
        +
        widgets.value-stepper.right-button-click
        + +
        +
        +
        widgets.button-state.disabled-state
        + +
        +
        +
        +
        widget-config.appearance
        + + + {{ valueStepperTypeTranslationMap.get(type) | translate }} + + +
        + + {{ 'widgets.value-stepper.auto-scale' | translate }} + +
        + +
        +
        {{ 'widgets.value-stepper.value-range' | translate }}
        +
        +
        widgets.value-stepper.min-range
        + + + +
        widgets.value-stepper.max-range
        + + + +
        +
        +
        +
        {{ 'widgets.value-stepper.value-increment-decrement-step' | translate }}
        + + + +
        +
        + + {{ 'widgets.value-stepper.value' | translate }} + +
        + + + +
        widget-config.decimals-suffix
        +         + + + + +
        +
        + +
        +
        {{ 'widgets.value-stepper.value-box-background' | translate }}
        + + +
        +
        + + {{ 'widgets.value-stepper.border' | translate }} + +
        + + +
        px
        +         + + +
        +
        +
        +
        +
        +
        widgets.value-stepper.button-appearance
        + + {{ 'widgets.value-stepper.left' | translate }} + {{ 'widgets.value-stepper.right' | translate }} + +
        +
        +
        + + {{ 'widgets.value-stepper.left-button' | translate }} + +
        +
        +
        {{ 'widgets.value-stepper.icon' | translate }}
        +
        + + + + + + +
        +
        +
        +
        {{ 'widgets.power-button.power-on-colors' | translate }}
        +
        +
        +
        widgets.power-button.main
        + + +
        + +
        +
        widgets.power-button.background
        + + +
        +
        +
        +
        +
        {{ 'widgets.power-button.disabled-colors' | translate }}
        +
        +
        +
        widgets.power-button.main
        + + +
        + +
        +
        widgets.power-button.background
        + + +
        +
        +
        +
        +
        +
        + + {{ 'widgets.value-stepper.right-button' | translate }} + +
        +
        +
        {{ 'widgets.value-stepper.icon' | translate }}
        +
        + + + + + + +
        +
        +
        +
        {{ 'widgets.power-button.power-on-colors' | translate }}
        +
        +
        +
        widgets.power-button.main
        + + +
        + +
        +
        widgets.power-button.background
        + + +
        +
        +
        +
        +
        {{ 'widgets.power-button.disabled-colors' | translate }}
        +
        +
        +
        widgets.power-button.main
        + + +
        + +
        +
        widgets.power-button.background
        + + +
        +
        +
        +
        +
        +         diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/value-stepper-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/value-stepper-widget-settings.component.ts new file mode 100644 index 0000000000..119c099da0 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/value-stepper-widget-settings.component.ts @@ -0,0 +1,191 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component } from '@angular/core'; +import { TargetDevice, WidgetSettings, WidgetSettingsComponent, widgetType } from '@shared/models/widget.models'; +import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { ValueType } from '@shared/models/constants'; +import { getTargetDeviceFromDatasources } from '@shared/models/widget-settings.models'; +import { + valueStepperDefaultSettings, + valueStepperTypeImages, + valueStepperTypes, + valueStepperTypeTranslations +} from '@home/components/widget/lib/rpc/value-stepper-widget.models'; +import { formatValue } from '@core/utils'; + +type ButtonAppearanceType = 'left' | 'right'; + +@Component({ + selector: 'tb-value-stepper-widget-settings', + templateUrl: './value-stepper-widget-settings.component.html', + styleUrls: ['../widget-settings.scss'] +}) +export class ValueStepperWidgetSettingsComponent extends WidgetSettingsComponent { + + get targetDevice(): TargetDevice { + const datasources = this.widgetConfig?.config?.datasources; + return getTargetDeviceFromDatasources(datasources); + } + + get widgetType(): widgetType { + return this.widgetConfig?.widgetType; + } + get borderRadius(): string { + return this.widgetConfig?.config?.borderRadius; + } + + valueType = ValueType; + + valueStepperWidgetSettingsForm: UntypedFormGroup; + + valueStepperTypeTranslationMap = valueStepperTypeTranslations; + valueStepperTypes = valueStepperTypes; + valueStepperTypeImageMap = valueStepperTypeImages; + + buttonAppearanceType: ButtonAppearanceType = 'left'; + + valuePreviewFn = this._valuePreviewFn.bind(this); + + constructor(protected store: Store, + private fb: UntypedFormBuilder) { + super(store); + } + + protected settingsForm(): UntypedFormGroup { + return this.valueStepperWidgetSettingsForm; + } + + protected defaultSettings(): WidgetSettings { + return {...valueStepperDefaultSettings}; + } + + protected onSettingsSet(settings: WidgetSettings) { + this.valueStepperWidgetSettingsForm = this.fb.group({ + initialState: [settings.initialState, []], + leftButtonClick: [settings.leftButtonClick, []], + rightButtonClick: [settings.rightButtonClick, []], + disabledState: [settings.disabledState, []], + + appearance: this.fb.group({ + type: [settings.appearance.type, []], + autoScale: [settings.appearance.autoScale, []], + minValueRange: [settings.appearance.minValueRange, []], + maxValueRange: [settings.appearance.maxValueRange, []], + valueStep: [settings.appearance.valueStep, [Validators.min(0)]], + showValueBox: [settings.appearance.showValueBox, []], + valueUnits: [settings.appearance.valueUnits, []], + valueDecimals: [settings.appearance.valueDecimals, []], + valueFont: [settings.appearance.valueFont, []], + valueColor: [settings.appearance.valueColor], + valueBoxBackground: [settings.appearance.valueBoxBackground, []], + showBorder: [settings.appearance.showBorder, []], + borderWidth: [settings.appearance.borderWidth, []], + borderColor: [settings.appearance.borderColor, []] + }), + + buttonAppearance: this.fb.group({ + leftButton: this.fb.group({ + showButton: [settings.buttonAppearance.leftButton.showButton], + icon: [settings.buttonAppearance.leftButton.icon], + iconSize: [settings.buttonAppearance.leftButton.iconSize], + iconSizeUnit: [settings.buttonAppearance.leftButton.iconSizeUnit], + mainColorOn: [settings.buttonAppearance.leftButton.mainColorOn, []], + backgroundColorOn: [settings.buttonAppearance.leftButton.backgroundColorOn, []], + mainColorDisabled: [settings.buttonAppearance.leftButton.mainColorDisabled, []], + backgroundColorDisabled: [settings.buttonAppearance.leftButton.backgroundColorDisabled, []] + }), + rightButton: this.fb.group({ + showButton: [settings.buttonAppearance.rightButton.showButton], + icon: [settings.buttonAppearance.rightButton.icon], + iconSize: [settings.buttonAppearance.rightButton.iconSize], + iconSizeUnit: [settings.buttonAppearance.rightButton.iconSizeUnit], + mainColorOn: [settings.buttonAppearance.rightButton.mainColorOn, []], + backgroundColorOn: [settings.buttonAppearance.rightButton.backgroundColorOn, []], + mainColorDisabled: [settings.buttonAppearance.rightButton.mainColorDisabled, []], + backgroundColorDisabled: [settings.buttonAppearance.rightButton.backgroundColorDisabled, []] + }) + }) + }); + } + + + protected validatorTriggers(): string[] { + return ['appearance.showValueBox', 'appearance.showBorder', + 'buttonAppearance.leftButton.showButton', 'buttonAppearance.rightButton.showButton']; + } + + protected updateValidators(_emitEvent: boolean): void { + const showValueBox: boolean = this.valueStepperWidgetSettingsForm.get('appearance').get('showValueBox').value; + const showBorder: boolean = this.valueStepperWidgetSettingsForm.get('appearance').get('showBorder').value; + const showLeftButton: boolean = this.valueStepperWidgetSettingsForm.get('buttonAppearance').get('leftButton').get('showButton').value; + const showRightButton: boolean = this.valueStepperWidgetSettingsForm.get('buttonAppearance').get('rightButton').get('showButton').value; + if (showValueBox) { + this.valueStepperWidgetSettingsForm.get('appearance').get('valueUnits').enable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('valueDecimals').enable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('valueFont').enable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('valueColor').enable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('valueBoxBackground').enable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('showBorder').enable({emitEvent: false}); + if (showBorder) { + this.valueStepperWidgetSettingsForm.get('appearance').get('borderWidth').enable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('borderColor').enable(); + } else { + this.valueStepperWidgetSettingsForm.get('appearance').get('borderWidth').disable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('borderColor').disable(); + } + } else { + this.valueStepperWidgetSettingsForm.get('appearance').get('valueUnits').disable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('valueDecimals').disable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('valueFont').disable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('valueColor').disable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('valueBoxBackground').disable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('showBorder').disable({emitEvent: false}); + this.valueStepperWidgetSettingsForm.get('appearance').get('borderWidth').disable(); + this.valueStepperWidgetSettingsForm.get('appearance').get('borderColor').disable(); + } + this.buttonValidators(showLeftButton, 'leftButton'); + this.buttonValidators(showRightButton, 'rightButton'); + } + + private buttonValidators(showButtonValue: boolean, button: string) { + if (showButtonValue) { + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('icon').enable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('iconSize').enable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('iconSizeUnit').enable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('mainColorOn').enable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('backgroundColorOn').enable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('mainColorDisabled').enable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('backgroundColorDisabled').enable() + } else { + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('icon').disable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('iconSize').disable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('iconSizeUnit').disable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('mainColorOn').disable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('backgroundColorOn').disable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('mainColorDisabled').disable() + this.valueStepperWidgetSettingsForm.get('buttonAppearance').get(button).get('backgroundColorDisabled').disable() + } + } + + private _valuePreviewFn(): string { + const units: string = this.valueStepperWidgetSettingsForm.get('appearance').get('valueUnits').value; + const decimals: number = this.valueStepperWidgetSettingsForm.get('appearance').get('valueDecimals').value; + return formatValue(48, decimals, units, false); + } +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.module.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.module.ts index 0aee57babb..e50fcb30f0 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.module.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.module.ts @@ -371,6 +371,9 @@ import { import { SegmentedButtonWidgetSettingsComponent } from '@home/components/widget/lib/settings/button/segmented-button-widget-settings.component'; +import { + ValueStepperWidgetSettingsComponent +} from '@home/components/widget/lib/settings/control/value-stepper-widget-settings.component'; @NgModule({ declarations: [ @@ -487,6 +490,7 @@ import { SingleSwitchWidgetSettingsComponent, ActionButtonWidgetSettingsComponent, SegmentedButtonWidgetSettingsComponent, + ValueStepperWidgetSettingsComponent, CommandButtonWidgetSettingsComponent, PowerButtonWidgetSettingsComponent, SliderWidgetSettingsComponent, @@ -624,6 +628,7 @@ import { SingleSwitchWidgetSettingsComponent, ActionButtonWidgetSettingsComponent, SegmentedButtonWidgetSettingsComponent, + ValueStepperWidgetSettingsComponent, CommandButtonWidgetSettingsComponent, PowerButtonWidgetSettingsComponent, SliderWidgetSettingsComponent, diff --git a/ui-ngx/src/app/modules/home/components/widget/widget-components.module.ts b/ui-ngx/src/app/modules/home/components/widget/widget-components.module.ts index 21f2d89dc6..fe752bb8b5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/widget-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/widget/widget-components.module.ts @@ -88,6 +88,7 @@ import { import { EllipsisChipListDirective } from '@shared/directives/ellipsis-chip-list.directive'; import { ScadaSymbolWidgetComponent } from '@home/components/widget/lib/scada/scada-symbol-widget.component'; import { TwoSegmentButtonWidgetComponent } from '@home/components/widget/lib/button/two-segment-button-widget.component'; +import { ValueStepperWidgetComponent } from '@home/components/widget/lib/rpc/value-stepper-widget.component'; @NgModule({ declarations: [ @@ -128,6 +129,7 @@ import { TwoSegmentButtonWidgetComponent } from '@home/components/widget/lib/but TwoSegmentButtonWidgetComponent, CommandButtonWidgetComponent, PowerButtonWidgetComponent, + ValueStepperWidgetComponent, SliderWidgetComponent, ToggleButtonWidgetComponent, TimeSeriesChartWidgetComponent, @@ -190,6 +192,7 @@ import { TwoSegmentButtonWidgetComponent } from '@home/components/widget/lib/but TwoSegmentButtonWidgetComponent, CommandButtonWidgetComponent, PowerButtonWidgetComponent, + ValueStepperWidgetComponent, SliderWidgetComponent, ToggleButtonWidgetComponent, TimeSeriesChartWidgetComponent, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 57df3865e9..5163a3c2bb 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -6617,6 +6617,8 @@ "layout-outlined-icon": "Outlined.Icon", "main": "Main", "background": "Background", + "button-icon-on": "Button icon 'On'", + "button-icon-off": "Button icon 'Off'", "power-on-colors": "Power 'On' colors", "power-off-colors": "Power 'Off' colors", "disabled-colors": "Disabled colors", @@ -6669,6 +6671,41 @@ "preview": "Preview", "copy-style-from": "Copy style from" }, + "value-stepper": { + "behavior": "Behavior", + "simplified": "Simplified", + "filled": "Filled", + "outlined": "Outlined", + "volume": "Volume", + "initial-state": "Initial state", + "initial-state-hint": "Action to get the initial value.", + "disabled-state": "Disabled state", + "disabled-state-hint": "Configure condition under which the component is disabled.", + "right-button-click": "Right button click", + "right-button-click-hint": "Action while pressing on right button.", + "left-button-click": "Left button click", + "left-button-click-hint": "Action while pressing on left button.", + "auto-scale": "Auto scale", + "value-range": "Range", + "min-range": "Min", + "max-range": "Max", + "value-increment-decrement-step": "Value increment/decrement step", + "value": "Value", + "value-box-background": "Value box background", + "border": "Border", + "button-appearance": "Button appearance", + "left": "Left", + "right": "Right", + "left-button": "Left button", + "right-button": "Right button", + "icon": "Icon", + "color-palette": "Color palette", + "main": "Main", + "background": "Background", + "button-icon-on": "Button icon 'On'", + "button-on-colors": "Power 'On' colors", + "disabled-colors": "Disabled colors" + }, "button-state": { "activated-state": "Activated state", "activated-state-hint": "Configure condition under which the button is active.", diff --git a/ui-ngx/src/assets/widget/value-stepper/filled.svg b/ui-ngx/src/assets/widget/value-stepper/filled.svg new file mode 100644 index 0000000000..26c94b92cd --- /dev/null +++ b/ui-ngx/src/assets/widget/value-stepper/filled.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/ui-ngx/src/assets/widget/value-stepper/simplified.svg b/ui-ngx/src/assets/widget/value-stepper/simplified.svg new file mode 100644 index 0000000000..8a71ad47c8 --- /dev/null +++ b/ui-ngx/src/assets/widget/value-stepper/simplified.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/ui-ngx/src/assets/widget/value-stepper/volume.svg b/ui-ngx/src/assets/widget/value-stepper/volume.svg new file mode 100644 index 0000000000..96bda9fa7a --- /dev/null +++ b/ui-ngx/src/assets/widget/value-stepper/volume.svg @@ -0,0 +1 @@ + \ No newline at end of file From 3d407618316610bb9ba681270350317a41025a30 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 7 Feb 2025 16:01:38 +0200 Subject: [PATCH 224/416] Remove outdated deprecation checks --- .../service/update/DeprecationService.java | 90 ------------------- .../notification/DefaultNotifications.java | 19 ---- .../rule/engine/api/NotificationCenter.java | 2 +- 3 files changed, 1 insertion(+), 110 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/update/DeprecationService.java diff --git a/application/src/main/java/org/thingsboard/server/service/update/DeprecationService.java b/application/src/main/java/org/thingsboard/server/service/update/DeprecationService.java deleted file mode 100644 index ff1f013953..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/update/DeprecationService.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.update; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.thingsboard.rule.engine.api.NotificationCenter; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.notification.info.GeneralNotificationInfo; -import org.thingsboard.server.common.data.notification.targets.platform.SystemAdministratorsFilter; -import org.thingsboard.server.dao.notification.DefaultNotifications; -import org.thingsboard.server.dao.notification.DefaultNotifications.DefaultNotification; -import org.thingsboard.server.queue.util.AfterStartUp; - -import java.util.Map; - -@Service -@Slf4j -@RequiredArgsConstructor -public class DeprecationService { - - private final NotificationCenter notificationCenter; - - @Value("${queue.type}") - private String queueType; - - @Value("${database.ts.type}") - private String tsType; - - @Value("${database.ts_latest.type}") - private String tsLatestType; - - @AfterStartUp(order = Integer.MAX_VALUE) - public void checkDeprecation() { - checkQueueTypeDeprecation(); - checkDatabaseTypeDeprecation(); - } - - private void checkQueueTypeDeprecation() { - String queueTypeName; - switch (queueType) { - case "aws-sqs" -> queueTypeName = "AWS SQS"; - case "pubsub" -> queueTypeName = "PubSub"; - case "service-bus" -> queueTypeName = "Azure Service Bus"; - case "rabbitmq" -> queueTypeName = "RabbitMQ"; - default -> { - return; - } - } - - log.warn("WARNING: Starting with ThingsBoard 4.0, {} will no longer be supported as a message queue for microservices. " + - "Please migrate to Apache Kafka. This change will not impact any rule nodes", queueTypeName); - sendNotification(DefaultNotifications.queueTypeDeprecation, Map.of( - "queueType", queueTypeName - )); - } - - private void checkDatabaseTypeDeprecation() { - String deprecatedDatabaseType = "Timescale"; - if (StringUtils.equalsAnyIgnoreCase(deprecatedDatabaseType, tsType, tsLatestType)) { - log.warn("WARNING: Starting with ThingsBoard 4.0, {} will no longer be supported as a storage provider. " + - "Please migrate to Cassandra or PostgreSQL.", deprecatedDatabaseType); - sendNotification(DefaultNotifications.databaseTypeDeprecation, Map.of( - "databaseType", deprecatedDatabaseType - )); - } - } - - private void sendNotification(DefaultNotification notification, Map info) { - notificationCenter.sendGeneralWebNotification(TenantId.SYS_TENANT_ID, new SystemAdministratorsFilter(), - notification.toTemplate(), new GeneralNotificationInfo(info)); - } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java index 16088e83c9..1a4c3ffab2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java @@ -372,25 +372,6 @@ public class DefaultNotifications { .build()) .build(); - public static final DefaultNotification queueTypeDeprecation = DefaultNotification.builder() - .name("Queue type deprecation") - .type(NotificationType.GENERAL) - .subject("WARNING: ${queueType} deprecation") - .text("Starting with ThingsBoard 4.0, ${queueType} will no longer be supported as a message queue for microservices. " + - "Please migrate to Apache Kafka. This change will not impact any rule nodes.") - .icon("warning").color(RED_COLOR) - .build(); - - public static final DefaultNotification databaseTypeDeprecation = DefaultNotification.builder() - .name("Database type deprecation") - .type(NotificationType.GENERAL) - .subject("WARNING: ${databaseType} deprecation") - .text("Starting with ThingsBoard 4.0, ${databaseType} will no longer be supported as a storage provider. " + - "Please migrate to Cassandra or PostgreSQL.") - .icon("warning") - .color(RED_COLOR) - .build(); - private final NotificationTemplateService templateService; private final NotificationRuleService ruleService; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/NotificationCenter.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/NotificationCenter.java index b8fb9b61ff..74c98cdec5 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/NotificationCenter.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/NotificationCenter.java @@ -36,7 +36,7 @@ public interface NotificationCenter { NotificationRequest processNotificationRequest(TenantId tenantId, NotificationRequest notificationRequest, FutureCallback callback); - void sendGeneralWebNotification(TenantId tenantId, UsersFilter recipients, NotificationTemplate template, GeneralNotificationInfo info); + void sendGeneralWebNotification(TenantId tenantId, UsersFilter recipients, NotificationTemplate template, GeneralNotificationInfo info); // for future use void sendSystemNotification(TenantId tenantId, NotificationTargetId targetId, NotificationType type, NotificationInfo info); // for future use and compatibility with PE From cfa7066ac62b602d32470ae61ae548604067a15e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 7 Feb 2025 16:23:38 +0200 Subject: [PATCH 225/416] Resolved comments --- ...culated-field-arguments-table.component.ts | 2 +- ...ulated-field-argument-panel.component.html | 4 ++-- ...ulated-field-argument-panel.component.scss | 22 ------------------- ...lculated-field-argument-panel.component.ts | 13 +++++------ 4 files changed, 9 insertions(+), 32 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 171c382b7a..3c51dc7f79 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -143,7 +143,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, entityName: this.entityName, - argumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argumentObj.argumentName), + usedArgumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argumentObj.argumentName), }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index a4f05b3288..4e9819a786 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -111,7 +111,7 @@
        } @else { - @if (isDeviceEntity) { + @if (enableAttributeScopeSelection) {
        {{ 'calculated-fields.attribute-scope' | translate }}
        @@ -155,7 +155,7 @@
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss deleted file mode 100644 index 45c17628d5..0000000000 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -:host ::ng-deep { - .time-window-field { - .mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field .mdc-notched-outline__notch { - border-left: 1px solid rgba(0, 0, 0, 0) !important; - } - } -} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 1632bd9311..a2e5545926 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -16,7 +16,7 @@ import { ChangeDetectorRef, Component, Input, OnInit, output } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; -import { FormBuilder, FormGroup, UntypedFormControl, ValidatorFn, Validators } from '@angular/forms'; +import { FormBuilder, FormControl, FormGroup, ValidatorFn, Validators } from '@angular/forms'; import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, @@ -42,7 +42,6 @@ import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', templateUrl: './calculated-field-argument-panel.component.html', - styleUrls: ['./calculated-field-argument-panel.component.scss'] }) export class CalculatedFieldArgumentPanelComponent implements OnInit { @@ -53,7 +52,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { @Input() tenantId: string; @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; - @Input() argumentNames: string[]; + @Input() usedArgumentNames: string[]; argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); @@ -111,7 +110,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { return this.argumentFormGroup.get('refEntityKey') as FormGroup; } - get isDeviceEntity(): boolean { + get enableAttributeScopeSelection(): boolean { return this.entityType === ArgumentEntityType.Device || (this.entityType === ArgumentEntityType.Current && (this.entityId.entityType === EntityType.DEVICE || this.entityId.entityType === EntityType.DEVICE_PROFILE)) @@ -196,16 +195,16 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { this.argumentFormGroup.get('refEntityId').get('id').setValue(''); this.argumentFormGroup.get('refEntityId') .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); - if (!this.isDeviceEntity) { + if (!this.enableAttributeScopeSelection) { this.refEntityKeyFormGroup.get('scope').setValue(AttributeScope.SERVER_SCOPE); } }); } private uniqNameRequired(): ValidatorFn { - return (control: UntypedFormControl) => { + return (control: FormControl) => { const newName = control.value.trim().toLowerCase(); - const isDuplicate = this.argumentNames?.some(name => name.toLowerCase() === newName); + const isDuplicate = this.usedArgumentNames?.some(name => name.toLowerCase() === newName); return isDuplicate ? { duplicateName: true } : null; }; From 0e1c3d66ff9a27cb00cfa3c7e0601a5723282468 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 7 Feb 2025 16:29:27 +0200 Subject: [PATCH 226/416] Added tb-form-panel-title tb-required --- .../dialog/calculated-field-dialog.component.html | 4 ++-- ui-ngx/src/form.scss | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 1d708f060a..8d69bf6539 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -65,7 +65,7 @@
        -
        {{ 'calculated-fields.arguments' | translate }}*
        +
        {{ 'calculated-fields.arguments' | translate }}
        -
        {{ 'calculated-fields.expression' | translate }}*
        +
        {{ 'calculated-fields.expression' | translate }}
        @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { diff --git a/ui-ngx/src/form.scss b/ui-ngx/src/form.scss index 0ae2b9584b..4c00a330c7 100644 --- a/ui-ngx/src/form.scss +++ b/ui-ngx/src/form.scss @@ -163,6 +163,13 @@ .tb-form-panel-title { font-weight: 500; font-size: 16px; + + &.tb-required::after { + font-size: 13px; + color: rgba(0, 0, 0, .54); + vertical-align: top; + content: " *"; + } } .tb-form-panel-hint { font-size: 12px; From 1cdbd2c338ceef94e2b09f161d3cdc605e080a55 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 7 Feb 2025 16:36:06 +0200 Subject: [PATCH 227/416] UI: fixed widget fqn --- .../src/main/data/json/system/widget_types/value_stepper.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/data/json/system/widget_types/value_stepper.json b/application/src/main/data/json/system/widget_types/value_stepper.json index 0126b25470..dea2406539 100644 --- a/application/src/main/data/json/system/widget_types/value_stepper.json +++ b/application/src/main/data/json/system/widget_types/value_stepper.json @@ -1,5 +1,5 @@ { - "fqn": "slider", + "fqn": "value_stepper", "name": "Value stepper", "deprecated": false, "image": "tb-image;/api/images/system/value-stepper-widget.svg", From 1e9d80d39b972ed4df1f406db25a5969d83fc88d Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 7 Feb 2025 16:37:51 +0200 Subject: [PATCH 228/416] Fixed mdc notch unnecessary border --- ui-ngx/src/form.scss | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui-ngx/src/form.scss b/ui-ngx/src/form.scss index 0ae2b9584b..c73a4fc9b4 100644 --- a/ui-ngx/src/form.scss +++ b/ui-ngx/src/form.scss @@ -398,6 +398,9 @@ border-color: rgba(0, 0, 0, 0.12); } } + .mdc-notched-outline__notch { + border-left: none; + } .mdc-floating-label { top: 20px; font-weight: 400; From c6f6408c22f6a724c6d7ed3ec1be68713e76ef05 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 17:04:01 +0200 Subject: [PATCH 229/416] refactored tests --- ...CalculatedFieldEntityMessageProcessor.java | 8 +- .../cf/CalculatedFieldIntegrationTest.java | 159 +++++++++--------- 2 files changed, 79 insertions(+), 88 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 0d327adca9..00913ec66d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

        + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

        - * http://www.apache.org/licenses/LICENSE-2.0 - *

        + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index fda0724b1b..f6dc09e3ce 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.cf; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Test; @@ -35,6 +34,8 @@ import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.controller.CalculatedFieldControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; @@ -53,12 +54,8 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes @Test public void testSimpleCalculatedField() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - - JsonNode timeSeries = JacksonUtil.toJsonNode("{\"temperature\":25}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, timeSeries); - - JsonNode attributes = JacksonUtil.toJsonNode("{\"deviceTemperature\":40}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"deviceTemperature\":40}")); CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(testDevice.getId()); @@ -72,15 +69,12 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Argument argument = new Argument(); ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); argument.setRefEntityKey(refEntityKey); - config.setArguments(Map.of("T", argument)); - config.setExpression("(T * 9/5) + 32"); Output output = new Output(); output.setName("fahrenheitTemp"); output.setType(OutputType.TIME_SERIES); - config.setOutput(output); calculatedField.setConfiguration(config); @@ -91,19 +85,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); - ObjectNode fahrenheitTemp = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); // update telemetry -> recalculate state - JsonNode newTelemetry = JacksonUtil.toJsonNode("{\"temperature\":30}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, newTelemetry); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); Thread.sleep(300); - ObjectNode fahrenheitTempAfterUpdate = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); - assertThat(fahrenheitTempAfterUpdate).isNotNull(); - assertThat(fahrenheitTempAfterUpdate.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); // update CF output -> perform calculation with updated output Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); @@ -114,21 +107,20 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); - ArrayNode temperatureF = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); assertThat(temperatureF).isNotNull(); assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); // update CF argument -> perform calculation with new argument - Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); Thread.sleep(300); - ArrayNode temperatureFAfterUpdateArg = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); - assertThat(temperatureFAfterUpdateArg).isNotNull(); - assertThat(temperatureFAfterUpdateArg.get(0).get("value").asText()).isEqualTo("104.0"); + temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); // update CF expression -> perform calculation with new expression savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); @@ -136,36 +128,23 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); - ArrayNode temperatureFAfterUpdateExpression = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); - assertThat(temperatureFAfterUpdateExpression).isNotNull(); - assertThat(temperatureFAfterUpdateExpression.get(0).get("value").asText()).isEqualTo("104.0"); + temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); } @Test public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - JsonNode deviceAttributes = JacksonUtil.toJsonNode("{\"x\":40}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, deviceAttributes); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":40}")); AssetProfile assetProfile = doPost("/api/assetProfile", createAssetProfile("Test Asset Profile"), AssetProfile.class); - Asset asset1 = new Asset(); - asset1.setName("Test asset 1"); - asset1.setAssetProfileId(assetProfile.getId()); - - Asset savedAsset1 = doPost("/api/asset", asset1, Asset.class); + Asset asset1 = createAsset("Test asset 1", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":11}")); - JsonNode asset1Attributes = JacksonUtil.toJsonNode("{\"y\":11}"); - doPost("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset1Attributes); - - Asset asset2 = new Asset(); - asset2.setName("Test asset 2"); - asset2.setAssetProfileId(assetProfile.getId()); - - Asset savedAsset2 = doPost("/api/asset", asset2, Asset.class); - - JsonNode asset2Attributes = JacksonUtil.toJsonNode("{\"y\":12}"); - doPost("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset2Attributes); + Asset asset2 = createAsset("Test asset 2", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":12}")); CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(assetProfile.getId()); @@ -205,62 +184,74 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); // result of asset 1 - ArrayNode z1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); assertThat(z1).isNotNull(); assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); // result of asset 2 - ArrayNode z2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); assertThat(z2).isNotNull(); assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); // update device telemetry -> recalculate state for all assets - JsonNode updatedDeviceAttributes = JacksonUtil.toJsonNode("{\"x\":25}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedDeviceAttributes); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); Thread.sleep(300); // result of asset 1 - ArrayNode updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); - assertThat(updZ1).isNotNull(); - assertThat(updZ1.get(0).get("value").asText()).isEqualTo("36.0"); + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); // result of asset 2 - ArrayNode updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); - assertThat(updZ2).isNotNull(); - assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); - -// // update asset 1 telemetry -> recalculate state only for asset 1 -// JsonNode updatedAsset1Attributes = JacksonUtil.toJsonNode("{\"x\":15}"); -// doPost("/api/plugins/telemetry/DEVICE/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset1Attributes); -// -// Thread.sleep(300); -// -// // result of asset 1 -// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ1).isNotNull(); -// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); -// -// // result of asset 2 (no changes) -// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ2).isNotNull(); -// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); -// -// // update asset 2 telemetry -> recalculate state only for asset 2 -// JsonNode updatedAsset2Attributes = JacksonUtil.toJsonNode("{\"x\":5}"); -// doPost("/api/plugins/telemetry/DEVICE/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset2Attributes); -// -// Thread.sleep(300); -// -// // result of asset 1 (no changes) -// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ1).isNotNull(); -// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); -// -// // result of asset 2 -// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ2).isNotNull(); -// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("30.0"); + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + + // update asset 1 telemetry -> recalculate state only for asset 1 + doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); + + Thread.sleep(300); + + // result of asset 1 + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + + // result of asset 2 (no changes) + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + + // update asset 2 telemetry -> recalculate state only for asset 2 + doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); + + Thread.sleep(300); + + // result of asset 1 (no changes) + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + + // result of asset 2 + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + + private ArrayNode getServerAttributes(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/attributes/SERVER_SCOPE?keys=" + String.join(",", keys), ArrayNode.class); + } + + private Asset createAsset(String name, AssetProfileId assetProfileId) { + Asset asset = new Asset(); + asset.setName(name); + asset.setAssetProfileId(assetProfileId); + return doPost("/api/asset", asset, Asset.class); } } From 44009d4e6cb9058b4489455a752a120ec803a193 Mon Sep 17 00:00:00 2001 From: Ekaterina Chantsova Date: Thu, 6 Feb 2025 19:50:36 +0200 Subject: [PATCH 230/416] UI: Date-time period - improve behaviour on changing start/end time --- .../time/datetime-period.component.html | 6 +- .../time/datetime-period.component.ts | 136 +++++++++++------- 2 files changed, 90 insertions(+), 52 deletions(-) diff --git a/ui-ngx/src/app/shared/components/time/datetime-period.component.html b/ui-ngx/src/app/shared/components/time/datetime-period.component.html index 84f6d29d68..03c5a6ac21 100644 --- a/ui-ngx/src/app/shared/components/time/datetime-period.component.html +++ b/ui-ngx/src/app/shared/components/time/datetime-period.component.html @@ -15,17 +15,17 @@ limitations under the License. --> -

        +
        datetime.from - + datetime.to - +
        diff --git a/ui-ngx/src/app/shared/components/time/datetime-period.component.ts b/ui-ngx/src/app/shared/components/time/datetime-period.component.ts index c0fbaea21e..0e212f7630 100644 --- a/ui-ngx/src/app/shared/components/time/datetime-period.component.ts +++ b/ui-ngx/src/app/shared/components/time/datetime-period.component.ts @@ -14,10 +14,17 @@ /// limitations under the License. /// -import { Component, forwardRef, Input, OnInit } from '@angular/core'; -import { ControlValueAccessor, NG_VALUE_ACCESSOR } from '@angular/forms'; -import { FixedWindow } from '@shared/models/time/time.models'; +import { Component, forwardRef, Input } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { DAY, FixedWindow, MINUTE } from '@shared/models/time/time.models'; import { MatFormFieldAppearance, SubscriptSizing } from '@angular/material/form-field'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { distinctUntilChanged } from 'rxjs/operators'; + +interface DateTimePeriod { + startDate: Date; + endDate: Date; +} @Component({ selector: 'tb-datetime-period', @@ -31,7 +38,7 @@ import { MatFormFieldAppearance, SubscriptSizing } from '@angular/material/form- } ] }) -export class DatetimePeriodComponent implements OnInit, ControlValueAccessor { +export class DatetimePeriodComponent implements ControlValueAccessor { @Input() disabled: boolean; @@ -41,25 +48,45 @@ export class DatetimePeriodComponent implements OnInit, ControlValueAccessor { @Input() appearance: MatFormFieldAppearance = 'fill'; - modelValue: FixedWindow; - - startDate: Date; - endDate: Date; - - endTime: any; + private modelValue: FixedWindow; maxStartDate: Date; - minEndDate: Date; maxEndDate: Date; - changePending = false; + private maxStartDateTs: number; + private minEndDateTs: number; + private maxStartTs: number; + private maxEndTs: number; - private propagateChange = null; + private timeShiftMs = MINUTE; - constructor() { - } + dateTimePeriodFormGroup = this.fb.group({ + startDate: this.fb.control(null), + endDate: this.fb.control(null) + }); + + private changePending = false; - ngOnInit(): void { + private propagateChange = null; + + constructor(private fb: FormBuilder) { + this.dateTimePeriodFormGroup.valueChanges.pipe( + distinctUntilChanged((prevDateTimePeriod, dateTimePeriod) => + prevDateTimePeriod.startDate === dateTimePeriod.startDate && prevDateTimePeriod.endDate === dateTimePeriod.endDate), + takeUntilDestroyed() + ).subscribe((dateTimePeriod: DateTimePeriod) => { + this.updateMinMaxDates(dateTimePeriod); + this.updateView(); + }); + + this.dateTimePeriodFormGroup.get('startDate').valueChanges.pipe( + distinctUntilChanged(), + takeUntilDestroyed() + ).subscribe(startDate => this.onStartDateChange(startDate)); + this.dateTimePeriodFormGroup.get('endDate').valueChanges.pipe( + distinctUntilChanged(), + takeUntilDestroyed() + ).subscribe(endDate => this.onEndDateChange(endDate)); } registerOnChange(fn: any): void { @@ -75,35 +102,38 @@ export class DatetimePeriodComponent implements OnInit, ControlValueAccessor { setDisabledState(isDisabled: boolean): void { this.disabled = isDisabled; + if (this.disabled) { + this.dateTimePeriodFormGroup.disable({emitEvent: false}); + } else { + this.dateTimePeriodFormGroup.enable({emitEvent: false}); + } } writeValue(datePeriod: FixedWindow): void { this.modelValue = datePeriod; if (this.modelValue) { - this.startDate = new Date(this.modelValue.startTimeMs); - this.endDate = new Date(this.modelValue.endTimeMs); + this.dateTimePeriodFormGroup.patchValue({ + startDate: new Date(this.modelValue.startTimeMs), + endDate: new Date(this.modelValue.endTimeMs) + }, {emitEvent: false}); } else { const date = new Date(); - this.startDate = new Date( - date.getFullYear(), - date.getMonth(), - date.getDate() - 1, - date.getHours(), - date.getMinutes(), - date.getSeconds(), - date.getMilliseconds()); - this.endDate = date; + this.dateTimePeriodFormGroup.patchValue({ + startDate: new Date(date.getTime() - DAY), + endDate: date + }, {emitEvent: false}); this.updateView(); } - this.updateMinMaxDates(); + this.updateMinMaxDates(this.dateTimePeriodFormGroup.value); } - updateView() { + private updateView() { let value: FixedWindow = null; - if (this.startDate && this.endDate) { + const dateTimePeriod = this.dateTimePeriodFormGroup.value; + if (dateTimePeriod.startDate && dateTimePeriod.endDate) { value = { - startTimeMs: this.startDate.getTime(), - endTimeMs: this.endDate.getTime() + startTimeMs: dateTimePeriod.startDate.getTime(), + endTimeMs: dateTimePeriod.endDate.getTime() }; } this.modelValue = value; @@ -114,32 +144,40 @@ export class DatetimePeriodComponent implements OnInit, ControlValueAccessor { } } - updateMinMaxDates() { - this.maxStartDate = new Date(this.endDate.getTime() - 1000); - this.minEndDate = new Date(this.startDate.getTime() + 1000); + private updateMinMaxDates(dateTimePeriod: Partial) { this.maxEndDate = new Date(); + this.maxEndTs = this.maxEndDate.getTime(); + this.maxStartTs = this.maxEndTs - this.timeShiftMs; + this.maxStartDate = new Date(this.maxStartTs); + + if (dateTimePeriod.endDate) { + this.maxStartDateTs = dateTimePeriod.endDate.getTime() - this.timeShiftMs; + } + if (dateTimePeriod.startDate) { + this.minEndDateTs = dateTimePeriod.startDate.getTime() + this.timeShiftMs; + } } - onStartDateChange() { - if (this.startDate) { - if (this.startDate.getTime() > this.maxStartDate.getTime()) { - this.startDate = new Date(this.maxStartDate.getTime()); + private onStartDateChange(startDate: Date) { + if (startDate) { + if (startDate.getTime() > this.maxStartTs) { + this.dateTimePeriodFormGroup.get('startDate').patchValue(new Date(this.maxStartTs), { emitEvent: false }); + } + if (startDate.getTime() > this.maxStartDateTs) { + this.dateTimePeriodFormGroup.get('endDate').patchValue(new Date(startDate.getTime() + this.timeShiftMs), { emitEvent: false }); } - this.updateMinMaxDates(); } - this.updateView(); } - onEndDateChange() { - if (this.endDate) { - if (this.endDate.getTime() < this.minEndDate.getTime()) { - this.endDate = new Date(this.minEndDate.getTime()); - } else if (this.endDate.getTime() > this.maxEndDate.getTime()) { - this.endDate = new Date(this.maxEndDate.getTime()); + private onEndDateChange(endDate: Date) { + if (endDate) { + if (endDate.getTime() > this.maxEndTs) { + this.dateTimePeriodFormGroup.get('endDate').patchValue(new Date(this.maxEndTs), { emitEvent: false }); + } + if (endDate.getTime() < this.minEndDateTs) { + this.dateTimePeriodFormGroup.get('startDate').patchValue(new Date(endDate.getTime() - this.timeShiftMs), { emitEvent: false }); } - this.updateMinMaxDates(); } - this.updateView(); } } From bdb44c9e86ba9a0fe2260811a39bfe75ab1b4324 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 7 Feb 2025 18:26:38 +0200 Subject: [PATCH 231/416] UI: Fixed mobile view in details page --- .../app/modules/home/pages/admin/queue/queue.component.html | 2 +- .../home/pages/admin/resource/js-resource.component.html | 4 ++-- .../pages/admin/resource/resources-library.component.html | 4 ++-- .../src/app/modules/home/pages/device/device.component.html | 2 +- ui-ngx/src/app/modules/home/pages/edge/edge.component.html | 2 +- .../modules/home/pages/ota-update/ota-update.component.html | 6 +++--- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/queue/queue.component.html b/ui-ngx/src/app/modules/home/pages/admin/queue/queue.component.html index f25cd1c857..68b73e7357 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/queue/queue.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/queue/queue.component.html @@ -22,7 +22,7 @@ [class.!hidden]="isEdit || isDetailsPage"> {{'common.open-details-page' | translate }} - - - - - - - - -
        diff --git a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.ts b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.ts index 417fc35bf4..e818e07ee4 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.ts @@ -14,24 +14,13 @@ /// limitations under the License. /// -import { - AfterViewInit, - Component, - ElementRef, - EventEmitter, - forwardRef, - Input, - OnInit, - Output, - ViewChild -} from '@angular/core'; +import { Component, ElementRef, EventEmitter, forwardRef, Input, OnInit, Output, ViewChild } from '@angular/core'; import { MatFormFieldAppearance, SubscriptSizing } from '@angular/material/form-field'; import { ControlValueAccessor, NG_VALUE_ACCESSOR, UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; -import { merge, Observable, of, Subject } from 'rxjs'; +import { firstValueFrom, merge, Observable, of, Subject } from 'rxjs'; import { catchError, debounceTime, map, share, switchMap, tap } from 'rxjs/operators'; import { Store } from '@ngrx/store'; import { AppState } from '@app/core/core.state'; -import { TranslateService } from '@ngx-translate/core'; import { AliasEntityType, EntityType } from '@shared/models/entity-type.models'; import { BaseData } from '@shared/models/base-data'; import { EntityId } from '@shared/models/id/entity-id'; @@ -51,22 +40,22 @@ import { coerceArray, coerceBoolean } from '@shared/decorators/coercion'; multi: true }] }) -export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit, AfterViewInit { +export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit { selectEntityFormGroup: UntypedFormGroup; - modelValue: string | EntityId | null; + private modelValue: string | EntityId | null; - entityTypeValue: EntityType | AliasEntityType; + private entityTypeValue: EntityType | AliasEntityType; - entitySubtypeValue: string; + private entitySubtypeValue: string; - entityText: string; + private entityText: string; noEntitiesMatchingText: string; notFoundEntities = 'entity.no-entities-text'; - entityRequiredText: string; + private entityRequiredText: string; filteredEntities: Observable>>; @@ -78,7 +67,7 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit private refresh$ = new Subject>>(); - private propagateChange = (v: any) => { }; + private propagateChange: (value: any) => void = () => { }; @Input() set entityType(entityType: EntityType) { @@ -166,7 +155,6 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit constructor(private store: Store, - public translate: TranslateService, private entityService: EntityService, private fb: UntypedFormBuilder) { this.selectEntityFormGroup = this.fb.group({ @@ -178,7 +166,7 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit this.propagateChange = fn; } - registerOnTouched(fn: any): void { + registerOnTouched(_fn: any): void { } ngOnInit() { @@ -188,7 +176,7 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit .pipe( debounceTime(150), tap(value => { - let modelValue; + let modelValue: string | EntityId; if (typeof value === 'string' || !value) { modelValue = null; } else { @@ -207,9 +195,7 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit ); } - ngAfterViewInit(): void {} - - load(): void { + private load(): void { if (this.entityTypeValue) { switch (this.entityTypeValue) { case EntityType.ASSET: @@ -327,7 +313,7 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit } } - getCurrentEntity(): BaseData | null { + private getCurrentEntity(): BaseData | null { const currentEntity = this.selectEntityFormGroup.get('entity').value; if (currentEntity && typeof currentEntity !== 'string') { return currentEntity as BaseData; @@ -359,16 +345,17 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit } let entity: BaseData = null; try { - entity = await this.entityService.getEntity(targetEntityType, id, {ignoreLoading: true, ignoreErrors: true}).toPromise(); + entity = await firstValueFrom(this.entityService.getEntity(targetEntityType, id, {ignoreLoading: true, ignoreErrors: true})); } catch (e) { this.propagateChange(null); } this.modelValue = entity !== null ? (this.useFullEntityId ? entity.id : entity.id.id) : null; - this.entityURL = getEntityDetailsPageURL(this.modelValue as string, targetEntityType); + this.entityURL = !entity ? '' : getEntityDetailsPageURL(entity.id.id, targetEntityType); this.selectEntityFormGroup.get('entity').patchValue(entity !== null ? entity : '', {emitEvent: false}); this.entityChanged.emit(entity); } else { this.modelValue = null; + this.entityURL = ''; this.selectEntityFormGroup.get('entity').patchValue('', {emitEvent: false}); } this.dirty = true; @@ -381,13 +368,14 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit } } - reset() { + private reset() { this.selectEntityFormGroup.get('entity').patchValue('', {emitEvent: false}); } - updateView(value: string | null, entity: BaseData | null) { + private updateView(value: string | EntityId | null, entity: BaseData | null) { if (!isEqual(this.modelValue, value)) { this.modelValue = value; + this.entityURL = !entity ? '' : getEntityDetailsPageURL(entity.id.id, entity.id.entityType as EntityType); this.propagateChange(this.modelValue); this.entityChanged.emit(entity); } @@ -397,7 +385,7 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit return entity ? entity.name : undefined; } - fetchEntities(searchText?: string): Observable>> { + private fetchEntities(searchText?: string): Observable>> { this.searchText = searchText; const targetEntityType = this.checkEntityType(this.entityTypeValue); return this.entityService.getEntitiesByNameFilter(targetEntityType, searchText, @@ -432,7 +420,7 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit }, 0); } - checkEntityType(entityType: EntityType | AliasEntityType): EntityType { + private checkEntityType(entityType: EntityType | AliasEntityType): EntityType { if (entityType === AliasEntityType.CURRENT_CUSTOMER) { return EntityType.CUSTOMER; } else if (entityType === AliasEntityType.CURRENT_TENANT) { @@ -454,4 +442,8 @@ export class EntityAutocompleteComponent implements ControlValueAccessor, OnInit $event.stopPropagation(); this.createNew.emit(); } + + get showEntityLink(): boolean { + return this.selectEntityFormGroup.get('entity').value && this.disabled && this.entityURL !== ''; + } } From 0cda72e6b2847c1baee498d2971bb3431b1d1895 Mon Sep 17 00:00:00 2001 From: devaskim Date: Sun, 9 Feb 2025 17:32:36 +0500 Subject: [PATCH 234/416] Encode username/passwrod pair instead of decoding it while creating request. --- .../java/org/thingsboard/rule/engine/rest/TbHttpClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index ed1ed4fa83..4745147de8 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -347,7 +347,7 @@ public class TbHttpClient { if (CredentialsType.BASIC == credentials.getType()) { BasicCredentials basicCredentials = (BasicCredentials) credentials; String authString = basicCredentials.getUsername() + ":" + basicCredentials.getPassword(); - String encodedAuthString = new String(Base64.getDecoder().decode(authString.getBytes(StandardCharsets.UTF_8))); + String encodedAuthString = new String(Base64.getEncoder().encode(authString.getBytes(StandardCharsets.UTF_8))); headers.add("Authorization", "Basic " + encodedAuthString); } } From 1256993569e75b0cb9c0fb72f04707b9d7900b7a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 10 Feb 2025 12:59:21 +0200 Subject: [PATCH 235/416] added new tests and fix proto serialization --- .../cf/CalculatedFieldIntegrationTest.java | 200 +++++++++++++++++- .../server/common/util/ProtoUtils.java | 4 +- 2 files changed, 201 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index f6dc09e3ce..91aad7b1cd 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -52,7 +52,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes } @Test - public void testSimpleCalculatedField() throws Exception { + public void testSimpleCalculatedFieldWhenAllTelemetryPresent() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"deviceTemperature\":40}")); @@ -69,6 +69,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Argument argument = new Argument(); ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); // not used because real telemetry value in db is present config.setArguments(Map.of("T", argument)); config.setExpression("(T * 9/5) + 32"); @@ -133,6 +134,99 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); } + @Test + public void testSimpleCalculatedFieldWhenNotAllTelemetryPresent() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> state is not ready -> no calculation performed + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + + // update telemetry -> perform calculation + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + Thread.sleep(300); + + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + } + + @Test + public void testSimpleCalculatedFieldWhenNotAllTelemetryPresentButDefaultValueIsSet() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> perform initial calculation with default value + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + + // update telemetry -> recalculate state + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + Thread.sleep(300); + + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + } + @Test public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); @@ -237,6 +331,110 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes z2 = getServerAttributes(asset2.getId(), "z"); assertThat(z2).isNotNull(); assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + + // add new entity to profile -> calculate state for new entity + Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); + + Thread.sleep(300); + + // result of asset 3 + ArrayNode z3 = getServerAttributes(asset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + + // update device telemetry -> recalculate state for all assets + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); + + Thread.sleep(300); + + // result of asset 1 + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); + + // result of asset 2 + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); + + // result of asset 3 + z3 = getServerAttributes(asset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + + // update profile for asset 3 -> delete state for asset 3 + AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); + asset3.setAssetProfileId(newAssetProfile.getId()); + asset3 = doPost("/api/asset", asset3, Asset.class); + + // update device telemetry -> recalculate state for asset 1 and asset 2 + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); + + Thread.sleep(300); + + // result of asset 1 + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); + + // result of asset 2 + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); + + // no changes for asset 3 + z3 = getServerAttributes(asset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + } + + @Test + public void testSimpleCalculatedFieldWhenExpressionIsInvalid() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); // not used because real telemetry value in db is present + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/0) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> ctx is not initialized -> no calculation perform + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + + // update telemetry -> ctx is not initialized -> no calculation perform + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + Thread.sleep(300); + + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); } private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index b81af2bfb9..07406977d2 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -127,8 +127,8 @@ public class ProtoUtils { builder.setProfileIdLSB(msg.getProfileId().getId().getLeastSignificantBits()); } if (msg.getOldProfileId() != null) { - builder.setProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); - builder.setProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); + builder.setOldProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } if (msg.getName() != null) { builder.setName(msg.getName()); From ec8e7a07268694685cfb047f1fc7724a36e50b2d Mon Sep 17 00:00:00 2001 From: Artem Barysh Date: Mon, 10 Feb 2025 14:28:46 +0200 Subject: [PATCH 236/416] Added versioning for ApiUsageState entity --- .../src/main/data/upgrade/basic/schema_update.sql | 2 ++ .../thingsboard/server/common/data/ApiUsageState.java | 5 ++++- .../server/dao/model/sql/ApiUsageStateEntity.java | 10 ++++------ dao/src/main/resources/sql/schema-entities.sql | 1 + 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 5d66056701..29b541327a 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -61,3 +61,5 @@ DO $$ $$; -- UPDATE SAVE TIME SERIES NODES END + +ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java index b13a9edd74..f6f3d82c09 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; @EqualsAndHashCode(callSuper = true) @Getter @Setter -public class ApiUsageState extends BaseData implements HasTenantId { +public class ApiUsageState extends BaseData implements HasTenantId, HasVersion { private static final long serialVersionUID = 8250339805336035966L; @@ -41,6 +41,7 @@ public class ApiUsageState extends BaseData implements HasTenan private ApiUsageStateValue emailExecState; private ApiUsageStateValue smsExecState; private ApiUsageStateValue alarmExecState; + private Long version; public ApiUsageState() { super(); @@ -62,6 +63,7 @@ public class ApiUsageState extends BaseData implements HasTenan this.emailExecState = ur.getEmailExecState(); this.smsExecState = ur.getSmsExecState(); this.alarmExecState = ur.getAlarmExecState(); + this.version = ur.getVersion(); } public boolean isTransportEnabled() { @@ -95,4 +97,5 @@ public class ApiUsageState extends BaseData implements HasTenan public boolean isAlarmCreationEnabled() { return alarmExecState != ApiUsageStateValue.DISABLED; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java index 43b99dd7ac..2a65029742 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.BaseVersionedEntity; import org.thingsboard.server.dao.model.ModelConstants; import java.util.UUID; @@ -40,7 +40,7 @@ import java.util.UUID; @EqualsAndHashCode(callSuper = true) @Entity @Table(name = ModelConstants.API_USAGE_STATE_TABLE_NAME) -public class ApiUsageStateEntity extends BaseSqlEntity implements BaseEntity { +public class ApiUsageStateEntity extends BaseVersionedEntity implements BaseEntity { @Column(name = ModelConstants.API_USAGE_STATE_TENANT_ID_COLUMN) private UUID tenantId; @@ -77,10 +77,7 @@ public class ApiUsageStateEntity extends BaseSqlEntity implements } public ApiUsageStateEntity(ApiUsageState ur) { - if (ur.getId() != null) { - this.setUuid(ur.getId().getId()); - } - this.setCreatedTime(ur.getCreatedTime()); + super(ur); if (ur.getTenantId() != null) { this.tenantId = ur.getTenantId().getId(); } @@ -116,6 +113,7 @@ public class ApiUsageStateEntity extends BaseSqlEntity implements ur.setEmailExecState(emailExecState); ur.setSmsExecState(smsExecState); ur.setAlarmExecState(alarmExecState); + ur.setVersion(version); return ur; } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 916a5487f8..71afc03081 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -702,6 +702,7 @@ CREATE TABLE IF NOT EXISTS api_usage_state ( email_exec varchar(32), sms_exec varchar(32), alarm_exec varchar(32), + version BIGINT DEFAULT 1, CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id) ); From 55f279944660e2c2d92825ba799c52719e3d529c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Mon, 10 Feb 2025 15:35:14 +0200 Subject: [PATCH 237/416] Implemented Calculated Fields debug test --- .../core/http/calculated-fields.service.ts | 5 + .../calculated-fields-table-config.ts | 34 +++- ...lated-field-arguments-table.component.html | 2 +- ...lculated-field-debug-dialog.component.html | 1 + ...calculated-field-debug-dialog.component.ts | 6 +- .../calculated-field-dialog.component.html | 8 + .../calculated-field-dialog.component.ts | 12 +- .../components/public-api.ts | 1 + ...ulated-field-test-arguments.component.html | 36 ++++ ...lculated-field-test-arguments.component.ts | 88 ++++++++++ ...ed-field-script-test-dialog.component.html | 101 ++++++++++++ ...ed-field-script-test-dialog.component.scss | 51 ++++++ ...ated-field-script-test-dialog.component.ts | 154 ++++++++++++++++++ .../components/event/event-table-config.ts | 10 ++ .../home/components/home-components.module.ts | 10 ++ .../shared/models/calculated-field.models.ts | 12 ++ ui-ngx/src/app/shared/models/entity.models.ts | 2 +- .../assets/locale/locale.constant-en_US.json | 6 + 18 files changed, 527 insertions(+), 12 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 9d8658f124..8e7ab6795e 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -22,6 +22,7 @@ import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; +import { TestScriptResult } from '@shared/models/rule-node.models'; @Injectable({ providedIn: 'root' @@ -48,4 +49,8 @@ export class CalculatedFieldsService { return this.http.get>(`/api/${entityType}/${id}/calculatedFields${pageLink.toQuery()}`, defaultHttpOptionsFromConfig(config)); } + + public testScript(inputParams: any, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 026c249159..8c00cbc925 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -38,9 +38,13 @@ import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDebugDialogData, - CalculatedFieldDialogData + CalculatedFieldDialogData, CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; -import { CalculatedFieldDebugDialogComponent, CalculatedFieldDialogComponent } from './components/public-api'; +import { + CalculatedFieldDebugDialogComponent, + CalculatedFieldDialogComponent, + CalculatedFieldScriptTestDialogComponent +} from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; @@ -53,7 +57,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, id), + action: (id?: CalculatedFieldId, expression?: string) => this.openDebugDialog.call(this, id, expression), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -134,10 +138,10 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog(id) + action: () => this.openDebugDialog(id, configuration?.expression) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -198,19 +202,22 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { tenantId: this.tenantId, entityId: this.entityId, - id + id, + expression, + testScriptFn: this.getTestScriptDialog.bind(this), } }) .afterClosed() @@ -251,4 +258,17 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } + + private getTestScriptDialog(argumentsObj: Record, expression: string, withApply = false): Observable { + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], + data: { + arguments: argumentsObj, + expression, + withApply, + } + }).afterClosed(); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index d8a6c7cdb0..5ca80b8214 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -24,7 +24,7 @@
        {{ 'entity.key' | translate }}
        -
        +
        @for (group of argumentsFormArray.controls; track group) {
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 91da675fea..f88176be8a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -34,6 +34,7 @@ [active]="true" [entityId]="data.id" [functionTestButtonLabel]="'common.test-function' | translate" + (debugEventSelected)="onDebugEventSelected($event)" />
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts index d81295948d..5b79476528 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -20,7 +20,7 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; -import { DebugEventType, EventType } from '@shared/models/event.models'; +import { CalculatedFieldEventBody, DebugEventType, EventType } from '@shared/models/event.models'; import { EventTableComponent } from '@home/components/event/event-table.component'; import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; @@ -51,4 +51,8 @@ export class CalculatedFieldDebugDialogComponent extends DialogComponent +
        + +
        }
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index d5bf243430..0613ccf1b7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -33,7 +33,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { map, startWith } from 'rxjs/operators'; +import { filter, map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -67,7 +67,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.data.additionalDebugActionConfig.action(this.data.value.id) + action: () => this.data.additionalDebugActionConfig.action(this.data.value.id, this.data.value.configuration.expression) } : null; readonly OutputTypeTranslations = OutputTypeTranslations; @@ -110,6 +110,14 @@ export class CalculatedFieldDialogComponent extends DialogComponent [k, ''])), + this.configFormGroup.get('expressionSCRIPT').value, + true + ).pipe(filter(Boolean)).subscribe((expression: string) => this.configFormGroup.get('expressionSCRIPT').setValue(expression)); + } + private applyDialogData(): void { const { configuration = {}, type = CalculatedFieldType.SIMPLE, ...value } = this.data.value ?? {}; const { expression, ...restConfig } = configuration as CalculatedFieldConfiguration; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts index 78b8862c2e..14ae73f7e1 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -18,3 +18,4 @@ export * from './dialog/calculated-field-dialog.component'; export * from './arguments-table/calculated-field-arguments-table.component'; export * from './panel/calculated-field-argument-panel.component'; export * from './debug-dialog/calculated-field-debug-dialog.component'; +export * from './test-dialog/calculated-field-script-test-dialog.component'; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html new file mode 100644 index 0000000000..e6b62a6862 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -0,0 +1,36 @@ + +
        +
        {{ 'calculated-fields.arguments' | translate }}
        +
        +
        +
        {{ 'calculated-fields.argument-name' | translate }}
        +
        {{ 'common.value' | translate }}
        +
        +
        + @for (group of argumentsFormArray.controls; track group) { +
        + + + + +
        + } +
        +
        +
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts new file mode 100644 index 0000000000..d7a33d1a89 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts @@ -0,0 +1,88 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef } from '@angular/core'; +import { + ControlValueAccessor, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + ValidationErrors, + FormBuilder, + FormGroup +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; + +@Component({ + selector: 'tb-calculated-field-test-arguments', + templateUrl: './calculated-field-test-arguments.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldTestArgumentsComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldTestArgumentsComponent), + multi: true, + } + ] +}) +export class CalculatedFieldTestArgumentsComponent extends PageComponent implements ControlValueAccessor, Validator { + + + argumentsFormArray = this.fb.array([]); + + private propagateChange: (value: { argumentName: string; value: unknown }) => void; + + constructor(private fb: FormBuilder) { + super(); + this.argumentsFormArray.valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(() => this.propagateChange(this.getValue())); + } + + registerOnChange(propagateChange: (value: { argumentName: string; value: unknown }) => void): void { + this.propagateChange = propagateChange; + } + + registerOnTouched(_): void { + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + Object.keys(argumentsObj).forEach(key => { + this.argumentsFormArray.push(this.fb.group({ + argumentName: [{ value: key, disabled: true}], + value: [argumentsObj[key]] + }) as FormGroup, {emitEvent: false}); + }); + } + + validate(): ValidationErrors | null { + return this.argumentsFormArray.valid ? null : { arguments: { valid: false } }; + } + + private getValue(): { argumentName: string; value: unknown } { + return this.argumentsFormArray.getRawValue().reduce((acc, rowItem) => { + const { argumentName, value } = rowItem; + acc[argumentName] = value; + return acc; + }, {}) as { argumentName: string; value: unknown }; + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html new file mode 100644 index 0000000000..ecfe2d41aa --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -0,0 +1,101 @@ + +
        + +

        {{ 'calculated-fields.test-script-function' | translate }} ({{ 'TBEL' }})

        + +         +
        +
        +
        +
        +
        +
        + {{ 'calculated-fields.expression' | translate }} +
        + +
        +
        +
        +
        +
        +
        + {{ 'calculated-fields.arguments' | translate }} +
        + +
        +
        +
        +
        +
        + common.output +
        + +
        +
        +
        +
        +
        +
        +
        + + + + @if (data.withApply) { + + } +
        +
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss new file mode 100644 index 0000000000..ee0d59b839 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .test-dialog-container { + .block-label { + padding: 4px; + color: #00acc1; + background: rgba(220, 220, 220, .35); + border-radius: 5px; + } + + .test-block-content { + padding-top: 5px; + padding-left: 5px; + border: 1px solid #c0c0c0; + } + } +} + +:host::ng-deep { + .test-dialog-container { + .gutter { + background-color: #eee; + background-repeat: no-repeat; + background-position: 50%; + } + + .gutter.gutter-horizontal { + cursor: col-resize; + background-image: url("../../../../../../../assets/split.js/grips/horizontal.png"); + } + + .gutter.gutter-vertical { + cursor: row-resize; + background-image: url("../../../../../../../assets/split.js/grips/vertical.png"); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts new file mode 100644 index 0000000000..63ee6fc5a7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -0,0 +1,154 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + Component, + DestroyRef, + ElementRef, + Inject, + ViewChild, +} from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder } from '@angular/forms'; +import { NEVER, Observable, of, switchMap } from 'rxjs'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { ContentType } from '@shared/models/constants'; +import { JsonContentComponent } from '@shared/components/json-content.component'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { beautifyJs } from '@shared/models/beautify.models'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-calculated-field-script-test-dialog', + templateUrl: './calculated-field-script-test-dialog.component.html', + styleUrls: ['./calculated-field-script-test-dialog.component.scss'], +}) +export class CalculatedFieldScriptTestDialogComponent extends DialogComponent implements AfterViewInit { + + @ViewChild('leftPanel', {static: true}) leftPanelElmRef: ElementRef; + @ViewChild('rightPanel', {static: true}) rightPanelElmRef: ElementRef; + @ViewChild('topRightPanel', {static: true}) topRightPanelElmRef: ElementRef; + @ViewChild('bottomRightPanel', {static: true}) bottomRightPanelElmRef: ElementRef; + + @ViewChild('expressionContent', {static: true}) expressionContent: JsonContentComponent; + + calculatedFieldScriptTestFormGroup = this.fb.group({ + expression: [], + arguments: [], + output: [] + }); + + readonly ContentType = ContentType; + readonly ScriptLanguage = ScriptLanguage; + readonly functionArgs = Object.keys(this.data.arguments); + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldScriptTestDialogData, + protected dialogRef: MatDialogRef, + private fb: FormBuilder, + private destroyRef: DestroyRef, + private calculatedFieldService: CalculatedFieldsService) { + super(store, router, dialogRef); + beautifyJs(this.data.expression, {indent_size: 4}).pipe(takeUntilDestroyed()).subscribe( + (res) => { + this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}); + } + ); + this.calculatedFieldScriptTestFormGroup.get('arguments').patchValue(this.data.arguments, {emitEvent: false}); + } + + ngAfterViewInit(): void { + this.initSplitLayout( + this.leftPanelElmRef.nativeElement, + this.rightPanelElmRef.nativeElement, + this.topRightPanelElmRef.nativeElement, + this.bottomRightPanelElmRef.nativeElement + ); + } + + cancel(): void { + this.dialogRef.close(null); + } + + onTestScript(): void { + this.testScript() + .pipe( + switchMap(output => beautifyJs(output, {indent_size: 4})), + takeUntilDestroyed(this.destroyRef) + ) + .subscribe(output => this.calculatedFieldScriptTestFormGroup.get('output').setValue(output)); + } + + save(): void { + this.testScript().pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.calculatedFieldScriptTestFormGroup.get('expression').markAsPristine(); + this.dialogRef.close(this.calculatedFieldScriptTestFormGroup.get('expression').value); + }); + } + + private testScript(): Observable { + if (this.checkInputParamErrors()) { + return this.calculatedFieldService.testScript({ + expression: this.calculatedFieldScriptTestFormGroup.get('expression').value, + arguments: this.calculatedFieldScriptTestFormGroup.get('arguments').value + }).pipe( + switchMap(result => { + if (result.error) { + this.store.dispatch(new ActionNotificationShow( + { + message: result.error, + type: 'error' + })); + return NEVER; + } else { + return of(result.output); + } + }), + ); + } else { + return NEVER; + } + } + + private checkInputParamErrors(): boolean { + this.expressionContent.validateOnSubmit(); + return !this.calculatedFieldScriptTestFormGroup.get('expression').invalid; + } + + private initSplitLayout(leftPanel, rightPanel, topRightPanel, bottomRightPanel): void { + Split([leftPanel, rightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'col-resize' + }); + + Split([topRightPanel, bottomRightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'row-resize', + direction: 'vertical' + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index a937e42bb8..c207e2d161 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -457,6 +457,16 @@ export class EventTableConfig extends EntityTableConfig { }); } break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.cellActionDescriptors.push({ + name: this.translate.instant('common.test-with-this-message', {test: this.translate.instant(this.testButtonLabel)}), + icon: 'bug_report', + isEnabled: () => true, + onAction: (_, entity) => { + this.debugEventSelected.next(entity.body); + } + }); + break; } this.getTable()?.cellActionDescriptorsUpdated(); } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index b7e35c5406..ea8bf7acf8 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -198,6 +198,12 @@ import { import { CalculatedFieldDebugDialogComponent } from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; +import { + CalculatedFieldScriptTestDialogComponent +} from '@home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component'; +import { + CalculatedFieldTestArgumentsComponent +} from '@home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component'; @NgModule({ declarations: @@ -347,6 +353,8 @@ import { CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, ], imports: [ CommonModule, @@ -490,6 +498,8 @@ import { CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 31dca236a4..e1f3ed8983 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -26,6 +26,7 @@ import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; import { AliasFilterType } from '@shared/models/alias.models'; +import { Observable } from 'rxjs'; export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId, ExportableEntity { debugSettings?: EntityDebugSettings; @@ -126,6 +127,8 @@ export interface CalculatedFieldArgumentValue extends CalculatedFieldArgument { argumentName: string; } +export type CalculatedFieldTestScriptFn = (argumentsObj: Record, expression: string, withApply?: boolean) => Observable; + export interface CalculatedFieldDialogData { value?: CalculatedField; buttonTitle: string; @@ -134,12 +137,21 @@ export interface CalculatedFieldDialogData { tenantId: string; entityName?: string; additionalDebugActionConfig: AdditionalDebugActionConfig; + testScriptFn: CalculatedFieldTestScriptFn; } export interface CalculatedFieldDebugDialogData { id?: CalculatedFieldId; entityId: EntityId; tenantId: string; + expression?: string; + testScriptFn: CalculatedFieldTestScriptFn; +} + +export interface CalculatedFieldScriptTestDialogData { + arguments: Record, + expression: string; + withApply: boolean; } export interface ArgumentEntityTypeParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index b9d2402850..1f73c0c8eb 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -204,7 +204,7 @@ export interface EntityDebugSettings { } export interface AdditionalDebugActionConfig { - action?: (id?: EntityId) => void; + action?: (id?: EntityId, ...restArguments: unknown[]) => void; title: string; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index d17409e52d..4822d23387 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -65,6 +65,7 @@ "next-with-label": "Next: {{label}}", "read-more": "Read more", "hide": "Hide", + "test": "Test", "done": "Done", "print": "Print", "restore": "Restore", @@ -1018,6 +1019,7 @@ "argument-name": "Argument name", "datasource": "Datasource", "add-argument": "Add argument", + "test-script-function": "Test script function", "no-arguments": "No arguments configured", "argument-settings": "Argument settings", "argument-current": "Current entity", @@ -1107,8 +1109,12 @@ "proceed": "Proceed", "open-details-page": "Open details page", "not-found": "Not found", + "value": "Value", "documentation": "Documentation", "time-left": "{{time}} left", + "output": "Output", + "test-function": "Test function", + "test-with-this-message": "{{test}} with this message", "suffix": { "s": "s", "ms": "ms" From 4cd5e0369d3312b8b2579dbef9fc8af73bf9bda2 Mon Sep 17 00:00:00 2001 From: devaskim Date: Sun, 9 Feb 2025 17:32:36 +0500 Subject: [PATCH 238/416] Encode username/passwrod pair instead of decoding it while creating request. (cherry picked from commit 0cda72e6b2847c1baee498d2971bb3431b1d1895) --- .../java/org/thingsboard/rule/engine/rest/TbHttpClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 566430bd8d..b4f5c7f096 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -343,7 +343,7 @@ public class TbHttpClient { if (CredentialsType.BASIC == credentials.getType()) { BasicCredentials basicCredentials = (BasicCredentials) credentials; String authString = basicCredentials.getUsername() + ":" + basicCredentials.getPassword(); - String encodedAuthString = new String(Base64.getDecoder().decode(authString.getBytes(StandardCharsets.UTF_8))); + String encodedAuthString = new String(Base64.getEncoder().encode(authString.getBytes(StandardCharsets.UTF_8))); headers.add("Authorization", "Basic " + encodedAuthString); } } From 5efb94dc7ae8f69d1d8e156f42ddb08fb72ca4f0 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 10 Feb 2025 15:58:01 +0200 Subject: [PATCH 239/416] Refactoring --- .../server/actors/ActorSystemContext.java | 6 +- .../server/actors/app/AppActor.java | 1 + .../CalculatedFieldEntityActor.java | 4 + ...CalculatedFieldEntityMessageProcessor.java | 20 +- .../CalculatedFieldManagerActor.java | 5 +- ...alculatedFieldManagerMessageProcessor.java | 15 +- .../server/actors/tenant/TenantActor.java | 1 + ... => CalculatedFieldProcessingService.java} | 16 +- .../cf/CalculatedFieldQueueService.java | 37 +++ .../CalculatedFieldStateService.java | 3 +- .../cf/DefaultCalculatedFieldInitService.java | 2 - ...aultCalculatedFieldProcessingService.java} | 287 +----------------- .../DefaultCalculatedFieldQueueService.java | 238 +++++++++++++++ .../CalculatedFieldEntityProfileCache.java | 1 + ...aultCalculatedFieldEntityProfileCache.java | 9 +- .../KafkaCalculatedFieldStateService.java | 25 +- .../RocksDBCalculatedFieldStateService.java | 2 +- ...faultTbCalculatedFieldConsumerService.java | 22 +- .../queue/DefaultTbClusterService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 12 +- .../server/common/msg/MsgType.java | 2 + .../cf/CalculatedFieldPartitionChangeMsg.java | 40 +++ .../queue/discovery/HashPartitionService.java | 5 + .../queue/discovery/PartitionService.java | 2 + 24 files changed, 408 insertions(+), 351 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/{CalculatedFieldExecutionService.java => CalculatedFieldProcessingService.java} (70%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java rename application/src/main/java/org/thingsboard/server/service/cf/{ctx => }/CalculatedFieldStateService.java (90%) rename application/src/main/java/org/thingsboard/server/service/cf/{DefaultCalculatedFieldExecutionService.java => DefaultCalculatedFieldProcessingService.java} (55%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index c714ed7dab..c787fc26af 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -106,9 +106,9 @@ import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -526,7 +526,7 @@ public class ActorSystemContext { @Lazy @Autowired(required = false) @Getter - private CalculatedFieldExecutionService calculatedFieldExecutionService; + private CalculatedFieldProcessingService calculatedFieldProcessingService; @Lazy @Autowired(required = false) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 8a6907c107..8c70eaf617 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -88,6 +88,7 @@ public class AppActor extends ContextAwareActor { case APP_INIT_MSG: break; case PARTITION_CHANGE_MSG: + case CF_PARTITIONS_CHANGE_MSG: ctx.broadcastToChildren(msg, true); break; case COMPONENT_LIFE_CYCLE_MSG: diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index cebe6b6a60..5e4aded41f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; @Slf4j public class CalculatedFieldEntityActor extends ContextAwareActor { @@ -50,6 +51,9 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { @Override protected boolean doProcess(TbActorMsg msg) { switch (msg.getMsgType()) { + case CF_PARTITIONS_CHANGE_MSG: + processor.process((CalculatedFieldPartitionChangeMsg) msg); + break; case CF_STATE_RESTORE_MSG: processor.process((CalculatedFieldStateRestoreMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 00913ec66d..61f7abb297 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -30,15 +30,16 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -67,8 +68,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM final TenantId tenantId; final EntityId entityId; - final CalculatedFieldExecutionService cfService; + final CalculatedFieldProcessingService cfService; final CalculatedFieldStateService cfStateService; + final int partition; TbActorCtx ctx; Map states = new HashMap<>(); @@ -77,14 +79,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM super(systemContext); this.tenantId = tenantId; this.entityId = entityId; - this.cfService = systemContext.getCalculatedFieldExecutionService(); + this.cfService = systemContext.getCalculatedFieldProcessingService(); this.cfStateService = systemContext.getCalculatedFieldStateService(); + this.partition = systemContext.getCalculatedFieldEntityProfileCache().getEntityIdPartition(tenantId, entityId); } void init(TbActorCtx ctx) { this.ctx = ctx; } + public void process(CalculatedFieldPartitionChangeMsg msg) { + if (!msg.getPartitions()[partition]) { + log.info("[{}][{}] Stopping entity actor due to change partition event.", partition, entityId); + ctx.stop(ctx.getSelf()); + } + } + public void process(CalculatedFieldStateRestoreMsg msg) { log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), msg.getId().cfId()); states.put(msg.getId().cfId(), msg.getState()); @@ -202,7 +212,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state != null) { return state; } else { - ListenableFuture stateFuture = systemContext.getCalculatedFieldExecutionService().fetchStateFromDb(ctx, entityId); + ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 22909dd5af..497602c93b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; /** * Created by ashvayka on 15.03.18. @@ -55,8 +56,8 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { @Override protected boolean doProcess(TbActorMsg msg) { switch (msg.getMsgType()) { - case PARTITION_CHANGE_MSG: - ctx.broadcastToChildren(msg, true); // TODO + case CF_PARTITIONS_CHANGE_MSG: + processor.onPartitionChange((CalculatedFieldPartitionChangeMsg) msg); break; case CF_INIT_MSG: processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index cdc31ed93c..22492f3c4f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -35,14 +35,15 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -52,6 +53,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -68,20 +70,20 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); - private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldProcessingService cfExecService; private final CalculatedFieldStateService cfStateService; private final CalculatedFieldEntityProfileCache cfEntityCache; private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; + protected final TenantId tenantId; protected TbActorCtx ctx; - final TenantId tenantId; CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); - this.cfExecService = systemContext.getCalculatedFieldExecutionService(); + this.cfExecService = systemContext.getCalculatedFieldProcessingService(); this.cfStateService = systemContext.getCalculatedFieldStateService(); this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); @@ -478,4 +480,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).remove(link)); } + public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { + ctx.broadcastToChildren(msg, true); + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 1b008f462a..16ea2c1398 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -170,6 +170,7 @@ public class TenantActor extends RuleChainManagerActor { case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: + case CF_PARTITIONS_CHANGE_MSG: case CF_ENTITY_LIFECYCLE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); break; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java similarity index 70% rename from application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java rename to application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 47fd560b4d..24b428593a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -15,15 +15,11 @@ */ package org.thingsboard.server.service.cf; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.rule.engine.api.AttributesSaveRequest; -import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -31,17 +27,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.List; -public interface CalculatedFieldExecutionService { - - /** - * Filter CFs based on the request entity. Push to the queue if any matching CF exist; - * - * @param request - telemetry save request; - * @param callback - */ - void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); - - void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); +public interface CalculatedFieldProcessingService { ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java new file mode 100644 index 0000000000..b84b54af81 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; + +import java.util.List; + +public interface CalculatedFieldQueueService { + + /** + * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * + * @param request - telemetry save request; + * @param callback + */ + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); + + void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java rename to application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index ce1562c735..37211c66c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cf.ctx; +package org.thingsboard.server.service.cf; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index 2afd6d8238..f8eb7e852e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -20,8 +20,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.asset.AssetService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java similarity index 55% rename from application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java rename to application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index e84e785a94..6b47053f2d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -32,25 +31,17 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.rule.engine.api.AttributesSaveRequest; -import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.OutputType; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -59,7 +50,6 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; @@ -67,12 +57,9 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; @@ -80,12 +67,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinke import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -93,149 +79,51 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; -import org.thingsboard.server.service.partition.AbstractPartitionBasedService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; -import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; -import java.util.function.Consumer; -import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; +@TbRuleEngineComponent @Service @Slf4j @RequiredArgsConstructor -public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { +public class DefaultCalculatedFieldProcessingService implements CalculatedFieldProcessingService { - public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - } - - @Override - public void onFailure(Throwable t) { - } - }; - - private final TbAssetProfileCache assetProfileCache; - private final TbDeviceProfileCache deviceProfileCache; - private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final TbClusterService clusterService; private final ApiLimitService apiLimitService; + private final PartitionService partitionService; - private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap states = new ConcurrentHashMap<>(); - - private static final Set supportedReferencedEntities = EnumSet.of( - EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT - ); - @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @PostConstruct public void init() { - super.init(); - calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); } @PreDestroy public void stop() { - super.stop(); - if (calculatedFieldExecutor != null) { - calculatedFieldExecutor.shutdownNow(); - } if (calculatedFieldCallbackExecutor != null) { calculatedFieldCallbackExecutor.shutdownNow(); } } - @Override - protected String getServiceName() { - return "Calculated Field Execution"; - } - - @Override - protected String getSchedulerExecutorName() { - return "calculated-field-scheduled"; - } - - @Override - public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { - var tenantId = request.getTenantId(); - var entityId = request.getEntityId(); - //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; - //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; - //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); - } - - @Override - public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { - var tenantId = request.getTenantId(); - var entityId = request.getEntityId(); - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); - } - - private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, - Predicate mainEntityFilter, Predicate linkedEntityFilter, - Supplier msg, FutureCallback callback) { - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); - if (send) { - clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); - } else { - if (callback != null) { - callback.onSuccess(null); - } - } - } - - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { - boolean send = false; - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); - if (!send) { - send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); - } - if (!send) { - send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId) - .map(calculatedFieldCache::getCalculatedFieldCtx) - .anyMatch(linkedEntityFilter); - } - } - return send; - } - @Override public ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { Map> argFutures = new HashMap<>(); @@ -262,21 +150,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - @Override - protected Map>> onAddedPartitions(Set addedPartitions) { - var result = new HashMap>>(); - return result; - } - - @Override - protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - cleanupEntity(entityId); - } - - private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); - } - @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { try { @@ -369,30 +242,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .build(); } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - necessaryArguments.forEach((key, argument) -> { - futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - }); - return Futures.transform(Futures.allAsList(futures), results -> { - onComplete.accept(argumentValues); - return null; - }, calculatedFieldCallbackExecutor); - } - - private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getRefEntityId(); - EntityId entityId = (argumentEntityId == null || isProfileEntity(argumentEntityId)) - ? targetEntityId - : argumentEntityId; - return fetchKvEntry(tenantId, entityId, argument); - } - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); @@ -462,78 +311,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } - private boolean isProfileEntity(EntityId entityId) { - return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); - } - - private EntityId getProfileId(TenantId tenantId, EntityId entityId) { - return switch (entityId.getEntityType()) { - case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - default -> null; - }; - } - - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { - ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); - List entries = request.getEntries(); - List versions = result.getVersions(); - for (int i = 0; i < entries.size(); i++) { - long tsVersion = versions.get(i); - TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); - telemetryMsg.addTsData(tsProto); - } - msg.setTelemetryMsg(telemetryMsg.build()); - - return msg.build(); - } - - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { - ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); - telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); - List entries = request.getEntries(); - for (int i = 0; i < entries.size(); i++) { - long attrVersion = versions.get(i); - AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); - telemetryMsg.addAttrData(attrProto); - } - msg.setTelemetryMsg(telemetryMsg.build()); - - return msg.build(); - } - - private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); - - telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - - telemetryMsg.setEntityType(entityId.getEntityType().name()); - telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - - if (calculatedFieldIds != null) { - for (CalculatedFieldId cfId : calculatedFieldIds) { - telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); - } - } - - if (tbMsgId != null) { - telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); - telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); - } - - if (tbMsgType != null) { - telemetryMsg.setTbMsgType(tbMsgType.name()); - } - - return telemetryMsg; - } - private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { return CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) @@ -541,60 +318,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .build(); } - private CalculatedFieldTelemetryUpdateRequest fromProto(CalculatedFieldTelemetryMsgProto proto) { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - - if (!proto.getTsDataList().isEmpty()) { - List updatedTelemetry = proto.getTsDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return new CalculatedFieldTimeSeriesUpdateRequest( - tenantId, entityId, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); - } else { - AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); - List updatedTelemetry = proto.getAttrDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return new CalculatedFieldAttributeUpdateRequest( - tenantId, entityId, scope, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); - } - } - - private static TbQueueCallback wrap(FutureCallback callback) { - if (callback != null) { - return new FutureCallbackWrapper(callback); - } else { - return DUMMY_TB_QUEUE_CALLBACK; - } - } - - private static class FutureCallbackWrapper implements TbQueueCallback { - private final FutureCallback callback; - - public FutureCallbackWrapper(FutureCallback callback) { - this.callback = callback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - callback.onSuccess(null); - } - - @Override - public void onFailure(Throwable t) { - callback.onFailure(t); - } - } - private static class TbCallbackWrapper implements TbQueueCallback { private final TbCallback callback; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java new file mode 100644 index 0000000000..9d1f9b6db5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -0,0 +1,238 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Predicate; +import java.util.function.Supplier; + +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueService { + + public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + } + }; + + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + private final CalculatedFieldCache calculatedFieldCache; + private final TbClusterService clusterService; + + private static final Set supportedReferencedEntities = EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT + ); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @Override + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; + //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; + //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + @Override + public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, + Predicate mainEntityFilter, Predicate linkedEntityFilter, + Supplier msg, FutureCallback callback) { + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); + if (send) { + clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); + } else { + if (callback != null) { + callback.onSuccess(null); + } + } + } + + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { + boolean send = false; + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); + if (!send) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); + } + if (!send) { + send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + .map(calculatedFieldCache::getCalculatedFieldCtx) + .anyMatch(linkedEntityFilter); + } + } + return send; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + List entries = request.getEntries(); + List versions = result.getVersions(); + for (int i = 0; i < entries.size(); i++) { + long tsVersion = versions.get(i); + TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); + telemetryMsg.addTsData(tsProto); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); + List entries = request.getEntries(); + for (int i = 0; i < entries.size(); i++) { + long attrVersion = versions.get(i); + AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); + telemetryMsg.addAttrData(attrProto); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); + + telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + + telemetryMsg.setEntityType(entityId.getEntityType().name()); + telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + + if (calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } + } + + if (tbMsgId != null) { + telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); + telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); + } + + if (tbMsgType != null) { + telemetryMsg.setTbMsgType(tbMsgType.name()); + } + + return telemetryMsg; + } + + private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + + private static TbQueueCallback wrap(FutureCallback callback) { + if (callback != null) { + return new FutureCallbackWrapper(callback); + } else { + return DUMMY_TB_QUEUE_CALLBACK; + } + } + + private static class FutureCallbackWrapper implements TbQueueCallback { + private final FutureCallback callback; + + public FutureCallbackWrapper(FutureCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java index 6b0718eb84..fec8c8ba12 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java @@ -32,4 +32,5 @@ public interface CalculatedFieldEntityProfileCache extends ApplicationListener

        getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); + int getEntityIdPartition(TenantId tenantId, EntityId entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 866cc86f24..9a56d35ac4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -42,7 +42,7 @@ import java.util.stream.Collectors; //TODO: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { - private static final Integer UNKNOWN = -1; + private static final Integer UNKNOWN = 0; private final ConcurrentMap tenantCache = new ConcurrentHashMap<>(); private final PartitionService partitionService; private volatile List myPartitions = Collections.emptyList(); @@ -84,4 +84,11 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent public Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); } + + @Override + public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { + var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + return tpi.getPartition().orElse(UNKNOWN); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 9d45532eec..a1999d438c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -17,34 +17,11 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.KvProtoUtil; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; - -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.UUID; -import java.util.stream.Collectors; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; @Service @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 86556d7adc..48e17954ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -37,7 +37,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import java.util.Map; import java.util.Optional; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 3d8bbddc80..f3e976084b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,6 +21,7 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @@ -32,8 +33,10 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -49,7 +52,6 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; @@ -58,6 +60,7 @@ import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -134,8 +137,23 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - log.debug("Subscribing to partitions: {}", event.getCalculatedFieldsPartitions()); + var partitions = event.getCalculatedFieldsPartitions(); + log.info("Subscribing to partitions: {}", partitions); + // TODO: @vklimov - before update of the main consumer, we should read the state topics and use + // CalculatedFieldStateService (KafkaCalculatedFieldStateService) to restore the states for entities that belong to new partitions. + // Cleanup entities that do not belong to current partition; mainConsumer.update(event.getCalculatedFieldsPartitions()); + // Cleanup old entities after corresponding consumers are stopped. + // Any periodic tasks need to check that the entity is still managed by the current server before processing. + actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + } + + private boolean[] partitionsToBooleanIndexArray(Set partitions) { + boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; + for(var tpi : partitions) { + tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); + } + return myPartitions; } private void processMsgs(List> msgs, TbQueueConsumer> consumer, CalculatedFieldQueueConfig config) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2a090fe701..48d26895b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -96,7 +96,7 @@ import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -148,7 +148,7 @@ public class DefaultTbClusterService implements TbClusterService { @Autowired @Lazy - private CalculatedFieldExecutionService calculatedFieldExecutionService; + private CalculatedFieldProcessingService calculatedFieldProcessingService; private final TopicService topicService; private final TbDeviceProfileCache deviceProfileCache; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 3f7f9c0b7d..b338a8382c 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -50,7 +50,7 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldQueueService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -77,7 +77,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; - private final CalculatedFieldExecutionService calculatedFieldExecutionService; + private final CalculatedFieldQueueService calculatedFieldQueueService; private ExecutorService tsCallBackExecutor; @@ -89,13 +89,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, - CalculatedFieldExecutionService calculatedFieldExecutionService) { + CalculatedFieldQueueService calculatedFieldQueueService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; - this.calculatedFieldExecutionService = calculatedFieldExecutionService; + this.calculatedFieldQueueService = calculatedFieldQueueService; } @PostConstruct @@ -147,7 +147,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } DonAsynchron.withCallback(resultFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { @@ -167,7 +167,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); DonAsynchron.withCallback(saveFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index e13e2d2ecb..9e5f00abf2 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -139,6 +139,8 @@ public enum MsgType { CF_INIT_MSG, // Sent to init particular calculated field; CF_LINK_INIT_MSG, // Sent to init particular calculated field; CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; + CF_PARTITIONS_CHANGE_MSG, // Sent when cluster event occures; + CF_ENTITY_LIFECYCLE_MSG, // Sent on CF/Device/Asset create/update/delete; CF_TELEMETRY_MSG, // Sent from queue to actor system; CF_LINKED_TELEMETRY_MSG, // Sent from queue to actor system; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java new file mode 100644 index 0000000000..5ab4b0ad9f --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +import java.util.Set; + +@Data +public class CalculatedFieldPartitionChangeMsg implements ToCalculatedFieldSystemMsg { + + private final boolean[] partitions; + + @Override + public TenantId getTenantId() { + return TenantId.SYS_TENANT_ID; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_PARTITIONS_CHANGE_MSG; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 5c6bf545b5..f2f1ccd19c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -526,6 +526,11 @@ public class HashPartitionService implements PartitionService { return list == null ? 0 : list.size(); } + @Override + public int getTotalCalculatedFieldPartitions() { + return cfPartitions; + } + private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 8e2152830e..b0e1229f97 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -77,4 +77,6 @@ public interface PartitionService { boolean isManagedByCurrentService(TenantId tenantId); + int getTotalCalculatedFieldPartitions(); + } From 84b9bde5772020787e4e842d1fd5d6971e35fc7d Mon Sep 17 00:00:00 2001 From: mpetrov Date: Mon, 10 Feb 2025 16:11:46 +0200 Subject: [PATCH 240/416] Refactoring --- ui-ngx/src/app/core/http/calculated-fields.service.ts | 6 +++--- ui-ngx/src/app/core/http/rule-chain.service.ts | 2 +- .../calculated-fields/calculated-fields-table-config.ts | 3 ++- .../calculated-field-debug-dialog.component.scss | 1 - .../dialog/calculated-field-dialog.component.ts | 5 ++++- .../calculated-field-script-test-dialog.component.html | 4 ++-- .../calculated-field-script-test-dialog.component.ts | 7 +++---- .../modules/home/components/event/event-table-config.ts | 8 +++----- ui-ngx/src/app/shared/models/calculated-field.models.ts | 7 +++++-- ui-ngx/src/app/shared/models/entity.models.ts | 5 +++++ ui-ngx/src/app/shared/models/rule-node.models.ts | 5 ----- 11 files changed, 28 insertions(+), 25 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 8e7ab6795e..acaf3b2817 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -19,10 +19,10 @@ import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; import { Observable } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; -import { CalculatedField } from '@shared/models/calculated-field.models'; +import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; -import { TestScriptResult } from '@shared/models/rule-node.models'; +import { TestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' @@ -50,7 +50,7 @@ export class CalculatedFieldsService { defaultHttpOptionsFromConfig(config)); } - public testScript(inputParams: any, config?: RequestConfig): Observable { + public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/core/http/rule-chain.service.ts b/ui-ngx/src/app/core/http/rule-chain.service.ts index e3353989cc..c1333df1ac 100644 --- a/ui-ngx/src/app/core/http/rule-chain.service.ts +++ b/ui-ngx/src/app/core/http/rule-chain.service.ts @@ -35,7 +35,6 @@ import { RuleNodeConfiguration, ScriptLanguage, TestScriptInputParams, - TestScriptResult } from '@app/shared/models/rule-node.models'; import { componentTypeBySelector, ResourcesService } from '../services/resources.service'; import { catchError, map, mergeMap } from 'rxjs/operators'; @@ -44,6 +43,7 @@ import { deepClone, snakeCase } from '@core/utils'; import { DebugRuleNodeEventBody } from '@app/shared/models/event.models'; import { Edge } from '@shared/models/edge.models'; import { IModulesMap } from '@modules/common/modules-map.models'; +import { TestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 8c00cbc925..c71eef8de3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -38,7 +38,8 @@ import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDebugDialogData, - CalculatedFieldDialogData, CalculatedFieldScriptTestDialogData + CalculatedFieldDialogData, + CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss index 23aa4070d7..19bf072b11 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -16,7 +16,6 @@ :host { .debug-dialog-container { height: 77vh; - min-width: 80vw; .debug-dialog-content { border-radius: 0; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 0613ccf1b7..20a37c8cc3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -115,7 +115,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent [k, ''])), this.configFormGroup.get('expressionSCRIPT').value, true - ).pipe(filter(Boolean)).subscribe((expression: string) => this.configFormGroup.get('expressionSCRIPT').setValue(expression)); + ).pipe(filter(Boolean)).subscribe((expression: string) => { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); } private applyDialogData(): void { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html index ecfe2d41aa..7af5e55b1c 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -48,10 +48,10 @@

        -
        +
        {{ 'calculated-fields.arguments' | translate }}
        - +
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts index 63ee6fc5a7..fa66827eb6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -37,6 +37,7 @@ import { beautifyJs } from '@shared/models/beautify.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; +import { filter } from 'rxjs/operators'; @Component({ selector: 'tb-calculated-field-script-test-dialog', @@ -71,10 +72,8 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent { - this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}); - } + beautifyJs(this.data.expression, {indent_size: 4}).pipe(filter(Boolean), takeUntilDestroyed()).subscribe( + (res) => this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}) ); this.calculatedFieldScriptTestFormGroup.get('arguments').patchValue(this.data.arguments, {emitEvent: false}); } diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index c207e2d161..39fb143de0 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -360,7 +360,7 @@ export class EventTableConfig extends EntityTableConfig { this.columns[1].width = '20%'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', - (entity) => `${entity.body.entityId.substring(0, 6)}…`, + (entity) => `${entity.body.entityId.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -380,7 +380,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '85px', - (entity) => `${entity.body.msgId?.substring(0, 6)}…`, + (entity) => `${entity.body.msgId?.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -462,9 +462,7 @@ export class EventTableConfig extends EntityTableConfig { name: this.translate.instant('common.test-with-this-message', {test: this.translate.instant(this.testButtonLabel)}), icon: 'bug_report', isEnabled: () => true, - onAction: (_, entity) => { - this.debugEventSelected.next(entity.body); - } + onAction: (_, entity) => this.debugEventSelected.next(entity.body) }); break; } diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index e1f3ed8983..1a208e122f 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -148,10 +148,13 @@ export interface CalculatedFieldDebugDialogData { testScriptFn: CalculatedFieldTestScriptFn; } -export interface CalculatedFieldScriptTestDialogData { +export interface CalculatedFieldScriptTestDialogData extends CalculatedFieldTestScriptInputParams { + withApply: boolean; +} + +export interface CalculatedFieldTestScriptInputParams { arguments: Record, expression: string; - withApply: boolean; } export interface ArgumentEntityTypeParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 1f73c0c8eb..3a8c06f544 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -203,6 +203,11 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } +export interface TestScriptResult { + output: string; + error: string; +} + export interface AdditionalDebugActionConfig { action?: (id?: EntityId, ...restArguments: unknown[]) => void; title: string; diff --git a/ui-ngx/src/app/shared/models/rule-node.models.ts b/ui-ngx/src/app/shared/models/rule-node.models.ts index a79fdbc376..ac08fa38ac 100644 --- a/ui-ngx/src/app/shared/models/rule-node.models.ts +++ b/ui-ngx/src/app/shared/models/rule-node.models.ts @@ -374,11 +374,6 @@ export interface TestScriptInputParams { msgType: string; } -export interface TestScriptResult { - output: string; - error: string; -} - export enum MessageType { POST_ATTRIBUTES_REQUEST = 'POST_ATTRIBUTES_REQUEST', POST_TELEMETRY_REQUEST = 'POST_TELEMETRY_REQUEST', From 3860d79613fb3b34f186558c9e4cbd246f22c270 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 10 Feb 2025 18:42:25 +0200 Subject: [PATCH 241/416] Improved Edge handling in case connect/disconnect. Speed up initial setup --- .../instructions/upgrade/docker/upgrade_db.md | 17 ++----------- .../service/edge/rpc/EdgeGrpcService.java | 3 +++ .../service/edge/rpc/EdgeGrpcSession.java | 8 ++++-- .../dashboard/BaseDashboardProcessor.java | 5 +++- .../edge/rpc/sync/EdgeRequestsService.java | 5 ++++ .../server/edge/DashboardEdgeTest.java | 25 ++++++++++++++++--- .../provider/KafkaMonolithQueueFactory.java | 2 +- .../provider/KafkaTbCoreQueueFactory.java | 2 +- 8 files changed, 43 insertions(+), 24 deletions(-) diff --git a/application/src/main/data/json/edge/instructions/upgrade/docker/upgrade_db.md b/application/src/main/data/json/edge/instructions/upgrade/docker/upgrade_db.md index a594ebe4e7..386c5f2743 100644 --- a/application/src/main/data/json/edge/instructions/upgrade/docker/upgrade_db.md +++ b/application/src/main/data/json/edge/instructions/upgrade/docker/upgrade_db.md @@ -43,19 +43,6 @@ volumes: Execute the following command to start upgrade process: ```bash -docker compose -f docker-compose-upgrade.yml up +docker compose -f docker-compose-upgrade.yml up --abort-on-container-exit {:copy-code} -``` - -Once upgrade process successfully completed, exit from the docker-compose shell by this combination: - -```text -Ctrl + C -``` - -Execute the following command to stop TB Edge upgrade container: - -```bash -docker compose -f docker-compose-upgrade.yml stop -{:copy-code} -``` +``` \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 73a504245d..471c47dafc 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -404,6 +404,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private void scheduleEdgeEventsCheck(EdgeGrpcSession session) { EdgeId edgeId = session.getEdge().getId(); TenantId tenantId = session.getEdge().getTenantId(); + + cancelScheduleEdgeEventsCheck(edgeId); + if (sessions.containsKey(edgeId)) { ScheduledFuture edgeEventCheckTask = edgeEventProcessingExecutorService.schedule(() -> { try { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 137c4d5bf5..b8cedf8a56 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -448,7 +448,11 @@ public abstract class EdgeGrpcSession implements Closeable { private void scheduleDownlinkMsgsPackSend(int attempt) { Runnable sendDownlinkMsgsTask = () -> { try { - if (isConnected() && !sessionState.getPendingMsgsMap().values().isEmpty()) { + if (!isConnected()) { + stopCurrentSendDownlinkMsgsTask(true); + return; + } + if (!sessionState.getPendingMsgsMap().values().isEmpty()) { List copy = new ArrayList<>(sessionState.getPendingMsgsMap().values()); if (attempt > 1) { String error = "Failed to deliver the batch"; @@ -525,11 +529,11 @@ public abstract class EdgeGrpcSession implements Closeable { private void onDownlinkResponse(DownlinkResponseMsg msg) { try { if (msg.getSuccess()) { - sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); log.debug("[{}][{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg); } else { log.error("[{}][{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg.getErrorMsg()); } + sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); if (sessionState.getPendingMsgsMap().isEmpty()) { log.debug("[{}][{}][{}] Pending msgs map is empty. Stopping current iteration", tenantId, edge.getId(), sessionId); stopCurrentSendDownlinkMsgsTask(false); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java index faf001fbba..39007552ad 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java @@ -41,7 +41,10 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { if (dashboard == null) { throw new RuntimeException("[{" + tenantId + "}] dashboardUpdateMsg {" + dashboardUpdateMsg + "} cannot be converted to dashboard"); } - Set newAssignedCustomers = new HashSet<>(dashboard.getAssignedCustomers()); + Set newAssignedCustomers = new HashSet<>(); + if (dashboard.getAssignedCustomers() != null && !dashboard.getAssignedCustomers().isEmpty()) { + newAssignedCustomers.addAll(dashboard.getAssignedCustomers()); + } Dashboard dashboardById = edgeCtx.getDashboardService().findDashboardById(tenantId, dashboardId); if (dashboardById == null) { created = true; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/EdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/EdgeRequestsService.java index 6d47c16680..e04d4d01af 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/EdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/EdgeRequestsService.java @@ -28,17 +28,22 @@ import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg; public interface EdgeRequestsService { + @Deprecated(since = "3.9.1", forRemoval = true) ListenableFuture processRuleChainMetadataRequestMsg(TenantId tenantId, Edge edge, RuleChainMetadataRequestMsg ruleChainMetadataRequestMsg); ListenableFuture processAttributesRequestMsg(TenantId tenantId, Edge edge, AttributesRequestMsg attributesRequestMsg); ListenableFuture processRelationRequestMsg(TenantId tenantId, Edge edge, RelationRequestMsg relationRequestMsg); + @Deprecated(since = "3.9.1", forRemoval = true) ListenableFuture processDeviceCredentialsRequestMsg(TenantId tenantId, Edge edge, DeviceCredentialsRequestMsg deviceCredentialsRequestMsg); + @Deprecated(since = "3.9.1", forRemoval = true) ListenableFuture processUserCredentialsRequestMsg(TenantId tenantId, Edge edge, UserCredentialsRequestMsg userCredentialsRequestMsg); + @Deprecated(since = "3.9.1", forRemoval = true) ListenableFuture processWidgetBundleTypesRequestMsg(TenantId tenantId, Edge edge, WidgetBundleTypesRequestMsg widgetBundleTypesRequestMsg); + @Deprecated(since = "3.9.1", forRemoval = true) ListenableFuture processEntityViewsRequestMsg(TenantId tenantId, Edge edge, EntityViewsRequestMsg entityViewsRequestMsg); } diff --git a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java index 5b09e05a09..42b75b5a74 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java @@ -48,12 +48,14 @@ public class DashboardEdgeTest extends AbstractEdgeTest { private static final int MOBILE_ORDER = 5; private static final String IMAGE = "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgd2lkdGg9IjQ4IiBoZWlnaHQ9IjQ4Ij48cGF0aCBkPSJNMTMuMjMgMTAuNTZWMTBjLTEuOTQgMC0zLjk5LjM5LTMuOTkgMi42NyAwIDEuMTYuNjEgMS45NSAxLjYzIDEuOTUuNzYgMCAxLjQzLS40NyAxLjg2LTEuMjIuNTItLjkzLjUtMS44LjUtMi44NG0yLjcgNi41M2MtLjE4LjE2LS40My4xNy0uNjMuMDYtLjg5LS43NC0xLjA1LTEuMDgtMS41NC0xLjc5LTEuNDcgMS41LTIuNTEgMS45NS00LjQyIDEuOTUtMi4yNSAwLTQuMDEtMS4zOS00LjAxLTQuMTcgMC0yLjE4IDEuMTctMy42NCAyLjg2LTQuMzggMS40Ni0uNjQgMy40OS0uNzYgNS4wNC0uOTNWNy41YzAtLjY2LjA1LTEuNDEtLjMzLTEuOTYtLjMyLS40OS0uOTUtLjctMS41LS43LTEuMDIgMC0xLjkzLjUzLTIuMTUgMS42MS0uMDUuMjQtLjI1LjQ4LS40Ny40OWwtMi42LS4yOGMtLjIyLS4wNS0uNDYtLjIyLS40LS41Ni42LTMuMTUgMy40NS00LjEgNi00LjEgMS4zIDAgMyAuMzUgNC4wMyAxLjMzQzE3LjExIDQuNTUgMTcgNi4xOCAxNyA3Ljk1djQuMTdjMCAxLjI1LjUgMS44MSAxIDIuNDguMTcuMjUuMjEuNTQgMCAuNzFsLTIuMDYgMS43OGgtLjAxIj48L3BhdGg+PHBhdGggZD0iTTIwLjE2IDE5LjU0QzE4IDIxLjE0IDE0LjgyIDIyIDEyLjEgMjJjLTMuODEgMC03LjI1LTEuNDEtOS44NS0zLjc2LS4yLS4xOC0uMDItLjQzLjI1LS4yOSAyLjc4IDEuNjMgNi4yNSAyLjYxIDkuODMgMi42MSAyLjQxIDAgNS4wNy0uNSA3LjUxLTEuNTMuMzctLjE2LjY2LjI0LjMyLjUxIj48L3BhdGg+PHBhdGggZD0iTTIxLjA3IDE4LjVjLS4yOC0uMzYtMS44NS0uMTctMi41Ny0uMDgtLjE5LjAyLS4yMi0uMTYtLjAzLS4zIDEuMjQtLjg4IDMuMjktLjYyIDMuNTMtLjMzLjI0LjMtLjA3IDIuMzUtMS4yNCAzLjMyLS4xOC4xNi0uMzUuMDctLjI2LS4xMS4yNi0uNjcuODUtMi4xNC41Ny0yLjV6Ij48L3BhdGg+PC9zdmc+"; + private static final String DASHBOARD_TITLE = "Edge Test Dashboard"; + @Test public void testDashboards() throws Exception { // create dashboard and assign to edge edgeImitator.expectMessageAmount(2); Dashboard dashboard = new Dashboard(); - dashboard.setTitle("Edge Test Dashboard"); + dashboard.setTitle(DASHBOARD_TITLE); dashboard.setMobileHide(true); dashboard.setImage(IMAGE); dashboard.setMobileOrder(MOBILE_ORDER); @@ -200,12 +202,27 @@ public class DashboardEdgeTest extends AbstractEdgeTest { Dashboard foundDashboard = doGet("/api/dashboard/" + dashboard.getUuidId(), Dashboard.class); Assert.assertNotNull(foundDashboard); - Assert.assertEquals("Edge Test Dashboard", foundDashboard.getName()); + Assert.assertEquals(DASHBOARD_TITLE, foundDashboard.getName()); PageData pageData = doGetTypedWithPageLink("/api/customer/" + savedCustomer.getId().toString() + "/dashboards?", new TypeReference<>() {}, new PageLink(100)); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals("Edge Test Dashboard", pageData.getData().get(0).getTitle()); + Assert.assertEquals(DASHBOARD_TITLE, pageData.getData().get(0).getTitle()); + + dashboard.setTitle(DASHBOARD_TITLE + " Updated"); + dashboard.setAssignedCustomers(null); + dashboardUpdateMsgBuilder.setEntity(JacksonUtil.toString(dashboard)); + dashboardUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + uplinkMsgBuilder = UplinkMsg.newBuilder(); + uplinkMsgBuilder.addDashboardUpdateMsg(dashboardUpdateMsgBuilder.build()); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + foundDashboard = doGet("/api/dashboard/" + dashboard.getUuidId(), Dashboard.class); + Assert.assertEquals(DASHBOARD_TITLE + " Updated", foundDashboard.getName()); } @Test @@ -256,7 +273,7 @@ public class DashboardEdgeTest extends AbstractEdgeTest { Dashboard dashboard = new Dashboard(); dashboard.setId(new DashboardId(UUID.randomUUID())); dashboard.setTenantId(tenantId); - dashboard.setTitle("Edge Test Dashboard"); + dashboard.setTitle(DASHBOARD_TITLE); dashboard.setAssignedCustomers(Sets.newHashSet(new ShortCustomerInfo(savedCustomer.getId(), savedCustomer.getTitle(), savedCustomer.isPublic()))); return dashboard; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index dd5d61e834..cd9d577b86 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -472,7 +472,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(topicService.buildTopicName("tb_edge_event.notifications." + tenantId + "." + edgeId)); - consumerBuilder.clientId("monolith-to-edge-event-consumer" + serviceInfoProvider.getServiceId()); + consumerBuilder.clientId("monolith-to-edge-event-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); consumerBuilder.groupId(topicService.buildTopicName("monolith-edge-event-consumer")); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeEventNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(edgeEventAdmin); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index cc0e044917..dffb127c70 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -421,7 +421,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(topicService.buildTopicName("tb_edge_event.notifications." + tenantId + "." + edgeId)); - consumerBuilder.clientId("tb-core-edge-event-consumer" + serviceInfoProvider.getServiceId()); + consumerBuilder.clientId("tb-core-edge-event-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); consumerBuilder.groupId(topicService.buildTopicName("tb-core-edge-event-consumer")); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeEventNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(edgeEventAdmin); From 1b4930ebafc27bd015fa432969235a56d8dfadce Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 10 Feb 2025 18:51:13 +0200 Subject: [PATCH 242/416] Introduced separate count for edge consumers --- .../server/queue/provider/KafkaMonolithQueueFactory.java | 3 ++- .../server/queue/provider/KafkaTbCoreQueueFactory.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index cd9d577b86..0767e5e303 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -96,6 +96,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin edgeEventAdmin; private final AtomicLong consumerCount = new AtomicLong(); + private final AtomicLong edgeConsumerCount = new AtomicLong(); public KafkaMonolithQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, @@ -472,7 +473,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(topicService.buildTopicName("tb_edge_event.notifications." + tenantId + "." + edgeId)); - consumerBuilder.clientId("monolith-to-edge-event-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.clientId("monolith-to-edge-event-consumer-" + serviceInfoProvider.getServiceId() + "-" + edgeConsumerCount.incrementAndGet()); consumerBuilder.groupId(topicService.buildTopicName("monolith-edge-event-consumer")); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeEventNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(edgeEventAdmin); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index dffb127c70..06fd6147a8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -95,6 +95,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin edgeEventAdmin; private final AtomicLong consumerCount = new AtomicLong(); + private final AtomicLong edgeConsumerCount = new AtomicLong(); public KafkaTbCoreQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -421,7 +422,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(topicService.buildTopicName("tb_edge_event.notifications." + tenantId + "." + edgeId)); - consumerBuilder.clientId("tb-core-edge-event-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.clientId("tb-core-edge-event-consumer-" + serviceInfoProvider.getServiceId() + "-" + edgeConsumerCount.incrementAndGet()); consumerBuilder.groupId(topicService.buildTopicName("tb-core-edge-event-consumer")); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeEventNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(edgeEventAdmin); From 8a662076c3c4e8927b9dd815c99d03fb7da0d4b9 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 11 Feb 2025 08:07:36 +0200 Subject: [PATCH 243/416] ack failed downlink if its not timeseries or attibutes failures --- .../server/service/edge/rpc/EdgeGrpcSession.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index b8cedf8a56..2e4d5c8aa5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -529,11 +529,16 @@ public abstract class EdgeGrpcSession implements Closeable { private void onDownlinkResponse(DownlinkResponseMsg msg) { try { if (msg.getSuccess()) { + sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); log.debug("[{}][{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg); } else { log.error("[{}][{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg.getErrorMsg()); + DownlinkMsg downlinkMsg = sessionState.getPendingMsgsMap().get(msg.getDownlinkMsgId()); + // if NOT timeseries or attributes failures - ack failed downlink + if (downlinkMsg.getEntityDataCount() == 0) { + sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); + } } - sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); if (sessionState.getPendingMsgsMap().isEmpty()) { log.debug("[{}][{}][{}] Pending msgs map is empty. Stopping current iteration", tenantId, edge.getId(), sessionId); stopCurrentSendDownlinkMsgsTask(false); From a315f18195188f7b592dce46d65e4dca90b5606b Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 11 Feb 2025 08:40:40 +0200 Subject: [PATCH 244/416] Improved GeneralEdgeEventFetcher for case when seqId less maxRecordsReadSize --- .../rpc/fetch/GeneralEdgeEventFetcher.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java index b2a1044981..93461f8c39 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java @@ -54,23 +54,25 @@ public class GeneralEdgeEventFetcher implements EdgeEventFetcher { log.trace("[{}] Finding general edge events [{}], seqIdStart = {}, pageLink = {}", tenantId, edge.getId(), seqIdStart, pageLink); PageData edgeEvents = edgeEventService.findEdgeEvents(tenantId, edge.getId(), seqIdStart, null, (TimePageLink) pageLink); - if (edgeEvents.getData().isEmpty()) { + if (!edgeEvents.getData().isEmpty()) { + return edgeEvents; + } + if (seqIdStart > this.maxReadRecordsCount) { edgeEvents = edgeEventService.findEdgeEvents(tenantId, edge.getId(), 0L, Math.max(this.maxReadRecordsCount, seqIdStart - this.maxReadRecordsCount), (TimePageLink) pageLink); if (edgeEvents.getData().stream().anyMatch(ee -> ee.getSeqId() < seqIdStart)) { log.info("[{}] seqId column of edge_event table started new cycle [{}]", tenantId, edge.getId()); this.seqIdNewCycleStarted = true; this.seqIdStart = 0L; - } else { - edgeEvents = new PageData<>(); - log.warn("[{}] unexpected edge notification message received. " + - "no new events found and seqId column of edge_event table doesn't started new cycle [{}]", tenantId, edge.getId()); + return edgeEvents; } } - return edgeEvents; + log.info("[{}] Unexpected edge notification message received. " + + "No new events found, and the seqId column of the edge_event table has not started a new cycle [{}].", tenantId, edge.getId()); + return new PageData<>(); } catch (Exception e) { - log.error("[{}] failed to find edge events [{}]", tenantId, edge.getId()); + log.error("[{}] Failed to find edge events [{}]", tenantId, edge.getId(), e); + return new PageData<>(); } - return new PageData<>(); } } From 227c7d7a1564ba802f6e05a5dcde14d67192a0a7 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 11 Feb 2025 11:44:49 +0200 Subject: [PATCH 245/416] Remove Timescale deprecation comment --- application/src/main/resources/thingsboard.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 4301f0f211..507024ae0f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -212,9 +212,9 @@ ui: database: ts_max_intervals: "${DATABASE_TS_MAX_INTERVALS:700}" # Max number of DB queries generated by a single API call to fetch telemetry records ts: - type: "${DATABASE_TS_TYPE:sql}" # cassandra or sql. timescale option is deprecated and will no longer be supported in ThingsBoard 4.0 + type: "${DATABASE_TS_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) ts_latest: - type: "${DATABASE_TS_LATEST_TYPE:sql}" # cassandra or sql. timescale option is deprecated and will no longer be supported in ThingsBoard 4.0 + type: "${DATABASE_TS_LATEST_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) # Cassandra driver configuration parameters cassandra: From d75bc3ac28e52ec44ef88ad5f238cb3cfac46cd9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 11 Feb 2025 12:47:37 +0200 Subject: [PATCH 246/416] Minor improvements --- .../cf/ctx/state/RocksDBCalculatedFieldStateService.java | 5 ++--- .../java/org/thingsboard/server/queue/util/AfterStartUp.java | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 48e17954ea..3374eb2660 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -47,7 +47,7 @@ import java.util.stream.Collectors; @Service @RequiredArgsConstructor -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) // Queue type in mem or Kafka; public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateService { private final ActorSystemContext actorSystemContext; @@ -66,12 +66,11 @@ public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateS restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); } - @Override public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { CalculatedFieldStateProto stateProto = toProto(stateId, state); long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); - if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { + if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= maxStateSizeInKBytes) { rocksDBService.put(toProto(stateId), stateProto); } callback.onSuccess(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index f97c3b0a1b..dea6d9ed9e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -41,7 +41,6 @@ public @interface AfterStartUp { int CF_READ_PROFILE_ENTITIES_SERVICE = 10; int CF_READ_CF_SERVICE = 11; int CF_STATE_RESTORE_SERVICE = 12; - int CF_CONSUMER_SERVICE = 13; int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; From 03e3391f6dc8b79f8050b02aaa4db500338f975a Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 11 Feb 2025 14:42:29 +0200 Subject: [PATCH 247/416] UI: Fixed incorrect liquid level widget config selector --- .../basic/indicator/liquid-level-card-basic-config.component.ts | 2 +- .../indicator/liquid-level-card-widget-settings.component.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/liquid-level-card-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/liquid-level-card-basic-config.component.ts index 3f7baa5d7c..0b06c062ef 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/liquid-level-card-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/liquid-level-card-basic-config.component.ts @@ -66,7 +66,7 @@ import { UtilsService } from '@core/services/utils.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @Component({ - selector: 'liquid-level-card-basic-config', + selector: 'tb-liquid-level-card-basic-config', templateUrl: './liquid-level-card-basic-config.component.html', styleUrls: ['../basic-config.scss'] }) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/liquid-level-card-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/liquid-level-card-widget-settings.component.ts index 70a163ac1a..11454392b4 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/liquid-level-card-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/liquid-level-card-widget-settings.component.ts @@ -56,7 +56,7 @@ import { EntityService } from '@core/http/entity.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @Component({ - selector: 'liquid-level-card-widget-settings', + selector: 'tb-liquid-level-card-widget-settings', templateUrl: './liquid-level-card-widget-settings.component.html', styleUrls: [] }) From 796a7027a7d04b5a0e2a98e3cf54a45ff179a792 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 11 Feb 2025 15:26:29 +0200 Subject: [PATCH 248/416] UI: Fixed incorrect shape layout settings in liquid level widget --- .../indicator/liquid-level-card-basic-config.component.html | 2 +- .../indicator/liquid-level-card-widget-settings.component.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/liquid-level-card-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/liquid-level-card-basic-config.component.html index 9808548bbe..a0776f5aa8 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/liquid-level-card-basic-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/liquid-level-card-basic-config.component.html @@ -63,7 +63,7 @@
        -
        +
        widgets.liquid-level-card.shape
        diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/liquid-level-card-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/liquid-level-card-widget-settings.component.html index a1aae64747..8d787116a8 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/liquid-level-card-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/liquid-level-card-widget-settings.component.html @@ -18,7 +18,7 @@
        -
        +
        widgets.liquid-level-card.shape
        From 9e19fab11dda76ebf65c88ac77c311ee175b6bc2 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 15:43:13 +0200 Subject: [PATCH 249/416] Review comments resolving and refactoring --- .../core/http/calculated-fields.service.ts | 6 +-- .../src/app/core/http/rule-chain.service.ts | 2 +- .../calculated-fields-table-config.ts | 44 ++++++++++--------- ...lculated-field-debug-dialog.component.html | 2 +- ...calculated-field-debug-dialog.component.ts | 10 +++-- .../calculated-field-dialog.component.html | 2 +- .../calculated-field-dialog.component.ts | 29 +++++++----- ...ed-field-script-test-dialog.component.html | 24 +++++----- ...ed-field-script-test-dialog.component.scss | 22 ++++++++++ ...ated-field-script-test-dialog.component.ts | 14 +++--- .../shared/models/calculated-field.models.ts | 15 +++---- ui-ngx/src/app/shared/models/entity.models.ts | 6 +-- .../src/app/shared/models/rule-node.models.ts | 4 +- .../en_US/calculated-field/expression_fn.md | 1 + .../calculated-field/test-expression_fn.md | 1 + 15 files changed, 109 insertions(+), 73 deletions(-) create mode 100644 ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md create mode 100644 ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index acaf3b2817..3e0e08f8e6 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -22,7 +22,7 @@ import { PageData } from '@shared/models/page/page-data'; import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; -import { TestScriptResult } from '@shared/models/entity.models'; +import { EntityTestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' @@ -50,7 +50,7 @@ export class CalculatedFieldsService { defaultHttpOptionsFromConfig(config)); } - public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { - return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); + public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/core/http/rule-chain.service.ts b/ui-ngx/src/app/core/http/rule-chain.service.ts index c1333df1ac..e3353989cc 100644 --- a/ui-ngx/src/app/core/http/rule-chain.service.ts +++ b/ui-ngx/src/app/core/http/rule-chain.service.ts @@ -35,6 +35,7 @@ import { RuleNodeConfiguration, ScriptLanguage, TestScriptInputParams, + TestScriptResult } from '@app/shared/models/rule-node.models'; import { componentTypeBySelector, ResourcesService } from '../services/resources.service'; import { catchError, map, mergeMap } from 'rxjs/operators'; @@ -43,7 +44,6 @@ import { deepClone, snakeCase } from '@core/utils'; import { DebugRuleNodeEventBody } from '@app/shared/models/event.models'; import { Edge } from '@shared/models/edge.models'; import { IModulesMap } from '@modules/common/modules-map.models'; -import { TestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index c71eef8de3..63b0fad732 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -34,12 +34,12 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; -import { catchError, filter, switchMap } from 'rxjs/operators'; +import { catchError, filter, switchMap, tap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDebugDialogData, CalculatedFieldDialogData, - CalculatedFieldScriptTestDialogData + CalculatedFieldTestScriptInputParams, } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, @@ -47,7 +47,6 @@ import { CalculatedFieldScriptTestDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; -import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -58,7 +57,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, id, expression), + action: (calculatedField: CalculatedField) => this.openDebugDialog.call(this, calculatedField), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -139,10 +138,11 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog(id, configuration?.expression) + action: () => this.openDebugDialog(calculatedField) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -178,8 +178,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField })), @@ -191,7 +191,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { + private getCalculatedFieldDialog(value?: CalculatedField, buttonTitle = 'action.add', isDirty = false): Observable { return this.dialog.open(CalculatedFieldDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], @@ -204,21 +204,20 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { tenantId: this.tenantId, - entityId: this.entityId, - id, - expression, - testScriptFn: this.getTestScriptDialog.bind(this), + value: calculatedField, + getTestScriptDialogFn: this.getTestScriptDialog.bind(this), } }) .afterClosed() @@ -260,16 +259,21 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } - private getTestScriptDialog(argumentsObj: Record, expression: string, withApply = false): Observable { - return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record): Observable { + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], data: { - arguments: argumentsObj, - expression, - withApply, + arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => ({...acc, [key]: '' }), {}), + expression: calculatedField.configuration.expression, } - }).afterClosed(); + }).afterClosed() + .pipe( + filter(Boolean), + tap(expression => + this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + ), + ); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index f88176be8a..8295a9c892 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -32,7 +32,7 @@ [disabledEventTypes]="[EventType.LC_EVENT, EventType.ERROR, EventType.STATS]" [defaultEventType]="DebugEventType.DEBUG_CALCULATED_FIELD" [active]="true" - [entityId]="data.id" + [entityId]="data.value.id" [functionTestButtonLabel]="'common.test-function' | translate" (debugEventSelected)="onDebugEventSelected($event)" /> diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts index 5b79476528..be27e277a6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -22,14 +22,14 @@ import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; import { CalculatedFieldEventBody, DebugEventType, EventType } from '@shared/models/event.models'; import { EventTableComponent } from '@home/components/event/event-table.component'; -import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; +import { CalculatedFieldDebugDialogData, CalculatedFieldType } from '@shared/models/calculated-field.models'; @Component({ selector: 'tb-calculated-field-debug-dialog', styleUrls: ['calculated-field-debug-dialog.component.scss'], templateUrl: './calculated-field-debug-dialog.component.html', }) -export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { +export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { @ViewChild(EventTableComponent, {static: true}) eventsTable: EventTableComponent; @@ -40,12 +40,13 @@ export class CalculatedFieldDebugDialogComponent extends DialogComponent, protected router: Router, @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDebugDialogData, - protected dialogRef: MatDialogRef) { + protected dialogRef: MatDialogRef) { super(store, router, dialogRef); } ngAfterViewInit(): void { this.eventsTable.entitiesTable.updateData(); + this.eventsTable.entitiesTable.cellActionDescriptors[0].isEnabled = () => this.data.value.type === CalculatedFieldType.SCRIPT; } cancel(): void { @@ -53,6 +54,7 @@ export class CalculatedFieldDebugDialogComponent extends DialogComponent this.dialogRef.close(expression)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index ad2d14cbbf..f50c552ec0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -99,7 +99,7 @@ [functionArgs]="functionArgs$ | async" [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" - helpId="[TODO]: [Calculated Fields] add valid link" + helpId="calculated-field/expression_fn" />
        - @if (data.withApply) { - - } +
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss index ee0d59b839..2420aa5a50 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -27,6 +27,28 @@ padding-left: 5px; border: 1px solid #c0c0c0; } + + .block-label-container { + position: absolute; + z-index: 10; + font-size: 12px; + font-weight: bold; + + &.left { + right: 112px; + top: 9px; + } + + &.right-bottom { + right: 40px; + top: 6px; + } + + &.right-top { + right: 8px; + top: 2px; + } + } } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts index fa66827eb6..bd19a68045 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -22,7 +22,7 @@ import { Inject, ViewChild, } from '@angular/core'; -import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { MAT_DIALOG_DATA, MatDialog, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { FormBuilder } from '@angular/forms'; @@ -36,8 +36,8 @@ import { ActionNotificationShow } from '@core/notification/notification.actions' import { beautifyJs } from '@shared/models/beautify.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; import { filter } from 'rxjs/operators'; +import { CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; @Component({ selector: 'tb-calculated-field-script-test-dialog', @@ -66,8 +66,9 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent, protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldScriptTestDialogData, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptInputParams, protected dialogRef: MatDialogRef, + private dialog: MatDialog, private fb: FormBuilder, private destroyRef: DestroyRef, private calculatedFieldService: CalculatedFieldsService) { @@ -101,13 +102,13 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent { + this.testScript(true).pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { this.calculatedFieldScriptTestFormGroup.get('expression').markAsPristine(); this.dialogRef.close(this.calculatedFieldScriptTestFormGroup.get('expression').value); }); } - private testScript(): Observable { + private testScript(onSave = false): Observable { if (this.checkInputParamErrors()) { return this.calculatedFieldService.testScript({ expression: this.calculatedFieldScriptTestFormGroup.get('expression').value, @@ -122,6 +123,9 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent, expression: string, withApply?: boolean) => Observable; +export type CalculatedFieldTestScriptFn = (calculatedField: CalculatedField, argumentsObj?: Record) => Observable; export interface CalculatedFieldDialogData { value?: CalculatedField; @@ -136,20 +136,15 @@ export interface CalculatedFieldDialogData { debugLimitsConfiguration: string; tenantId: string; entityName?: string; - additionalDebugActionConfig: AdditionalDebugActionConfig; + additionalDebugActionConfig: AdditionalDebugActionConfig<(calculatedField: CalculatedField) => void>; testScriptFn: CalculatedFieldTestScriptFn; + isDirty?: boolean; } export interface CalculatedFieldDebugDialogData { - id?: CalculatedFieldId; - entityId: EntityId; tenantId: string; - expression?: string; - testScriptFn: CalculatedFieldTestScriptFn; -} - -export interface CalculatedFieldScriptTestDialogData extends CalculatedFieldTestScriptInputParams { - withApply: boolean; + value: CalculatedField; + getTestScriptDialogFn: CalculatedFieldTestScriptFn; } export interface CalculatedFieldTestScriptInputParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 3a8c06f544..472d28f849 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -203,13 +203,13 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } -export interface TestScriptResult { +export interface EntityTestScriptResult { output: string; error: string; } -export interface AdditionalDebugActionConfig { - action?: (id?: EntityId, ...restArguments: unknown[]) => void; +export interface AdditionalDebugActionConfig void> { + action?: Action; title: string; } diff --git a/ui-ngx/src/app/shared/models/rule-node.models.ts b/ui-ngx/src/app/shared/models/rule-node.models.ts index ac08fa38ac..3e1eb70835 100644 --- a/ui-ngx/src/app/shared/models/rule-node.models.ts +++ b/ui-ngx/src/app/shared/models/rule-node.models.ts @@ -27,7 +27,7 @@ import { AppState } from '@core/core.state'; import { AbstractControl, UntypedFormGroup } from '@angular/forms'; import { RuleChainType } from '@shared/models/rule-chain.models'; import { DebugRuleNodeEventBody } from '@shared/models/event.models'; -import { HasEntityDebugSettings } from '@shared/models/entity.models'; +import { EntityTestScriptResult, HasEntityDebugSettings } from '@shared/models/entity.models'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; export interface RuleNodeConfiguration { @@ -374,6 +374,8 @@ export interface TestScriptInputParams { msgType: string; } +export type TestScriptResult = EntityTestScriptResult; + export enum MessageType { POST_ATTRIBUTES_REQUEST = 'POST_ATTRIBUTES_REQUEST', POST_TELEMETRY_REQUEST = 'POST_TELEMETRY_REQUEST', diff --git a/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md new file mode 100644 index 0000000000..f8173dc528 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md @@ -0,0 +1 @@ + diff --git a/ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md b/ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md new file mode 100644 index 0000000000..f8173dc528 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md @@ -0,0 +1 @@ + From 2833164968252d1011a3d64d53dbb3d73650c106 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 15:48:45 +0200 Subject: [PATCH 250/416] Refactoring --- .../calculated-fields/calculated-fields-table-config.ts | 2 +- .../components/dialog/calculated-field-dialog.component.ts | 2 +- ui-ngx/src/app/shared/models/calculated-field.models.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 63b0fad732..544860cca2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -203,7 +203,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { this.configFormGroup.get('expressionSCRIPT').setValue(expression); this.configFormGroup.get('expressionSCRIPT').markAsDirty(); diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index cec8fe5cd6..19b53bc91e 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -137,7 +137,7 @@ export interface CalculatedFieldDialogData { tenantId: string; entityName?: string; additionalDebugActionConfig: AdditionalDebugActionConfig<(calculatedField: CalculatedField) => void>; - testScriptFn: CalculatedFieldTestScriptFn; + getTestScriptDialogFn: CalculatedFieldTestScriptFn; isDirty?: boolean; } From f8d88387fa78bb7f8e93efbe759f8e9a40cad9d3 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 16:04:23 +0200 Subject: [PATCH 251/416] refactoring --- .../calculated-fields/calculated-fields-table-config.ts | 5 +++-- .../dialog/calculated-field-dialog.component.ts | 8 ++------ .../modules/home/components/event/event-table-config.ts | 4 ++-- ui-ngx/src/app/shared/models/entity.models.ts | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 544860cca2..318ffaca66 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -205,7 +205,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig ({...acc, [key]: '' }), {}), + arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}), expression: calculatedField.configuration.expression, } }).afterClosed() diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 8574ee99c2..04e1b539dd 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -33,7 +33,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { filter, map, startWith } from 'rxjs/operators'; +import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -121,11 +121,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent { - this.configFormGroup.get('expressionSCRIPT').setValue(expression); - this.configFormGroup.get('expressionSCRIPT').markAsDirty(); - }); + this.data.getTestScriptDialogFn(this.fromGroupValue).subscribe(); } private applyDialogData(): void { diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 39fb143de0..98d2fbefaf 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -360,7 +360,7 @@ export class EventTableConfig extends EntityTableConfig { this.columns[1].width = '20%'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', - (entity) => `${entity.body.entityId.substring(0, 8)}…`, + (entity) => `${entity.body.entityId.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -380,7 +380,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '85px', - (entity) => `${entity.body.msgId?.substring(0, 8)}…`, + (entity) => `${entity.body.msgId?.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 472d28f849..20a0ccbe77 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -209,7 +209,7 @@ export interface EntityTestScriptResult { } export interface AdditionalDebugActionConfig void> { - action?: Action; + action: Action; title: string; } From 9589317e9750bea3190798c8060deb522ada1e85 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 16:25:01 +0200 Subject: [PATCH 252/416] preserve edit dialog on test --- .../calculated-fields-table-config.ts | 15 +++++++++------ .../dialog/calculated-field-dialog.component.ts | 5 ++++- ...lculated-field-script-test-dialog.component.ts | 6 +++--- .../app/shared/models/calculated-field.models.ts | 6 +++++- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 318ffaca66..966ac5f008 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -39,7 +39,7 @@ import { CalculatedField, CalculatedFieldDebugDialogData, CalculatedFieldDialogData, - CalculatedFieldTestScriptInputParams, + CalculatedFieldTestScriptDialogData, } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, @@ -260,21 +260,24 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } - private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record): Observable { - return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record, openCalculatedFieldEdit = true): Observable { + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], data: { arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}), expression: calculatedField.configuration.expression, + openCalculatedFieldEdit } }).afterClosed() .pipe( filter(Boolean), - tap(expression => - this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) - ), + tap(expression => { + if (openCalculatedFieldEdit) { + this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + } + }), ); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 04e1b539dd..e75c962268 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -121,7 +121,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); } private applyDialogData(): void { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts index bd19a68045..94ffeeea52 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -37,7 +37,7 @@ import { beautifyJs } from '@shared/models/beautify.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { filter } from 'rxjs/operators'; -import { CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; +import { CalculatedFieldTestScriptDialogData } from '@shared/models/calculated-field.models'; @Component({ selector: 'tb-calculated-field-script-test-dialog', @@ -66,7 +66,7 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent, protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptInputParams, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptDialogData, protected dialogRef: MatDialogRef, private dialog: MatDialog, private fb: FormBuilder, @@ -123,7 +123,7 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent) => Observable; +export type CalculatedFieldTestScriptFn = (calculatedField: CalculatedField, argumentsObj?: Record, closeAllOnSave?: boolean) => Observable; export interface CalculatedFieldDialogData { value?: CalculatedField; @@ -152,6 +152,10 @@ export interface CalculatedFieldTestScriptInputParams { expression: string; } +export interface CalculatedFieldTestScriptDialogData extends CalculatedFieldTestScriptInputParams { + openCalculatedFieldEdit?: boolean; +} + export interface ArgumentEntityTypeParams { title: string; entityType: EntityType From 6eeaecf88ff69c5ca5de32867dc29e0be6d1456e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 16:30:24 +0200 Subject: [PATCH 253/416] close debug panel on additional action --- .../entity/debug/entity-debug-settings-panel.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html index dcc3355890..f7fd9f737b 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html @@ -54,7 +54,7 @@ } From 934a8daa4ea59b6132afcd48fd781f9812c512dc Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 17:29:44 +0200 Subject: [PATCH 254/416] only last arguments fix --- .../calculated-fields/calculated-fields-table-config.ts | 9 ++++++++- .../modules/home/components/event/event-table-config.ts | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 966ac5f008..9560f264b5 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -261,12 +261,19 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig, openCalculatedFieldEdit = true): Observable { + const emptyArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}); + const filledArguments = Object.keys(argumentsObj ?? {}).reduce((acc, key) => { + if (emptyArguments.hasOwnProperty(key)) { + acc[key] = argumentsObj[key]; + } + return acc; + }, {}); return this.dialog.open(CalculatedFieldScriptTestDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], data: { - arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}), + arguments: { ...emptyArguments, ...filledArguments }, expression: calculatedField.configuration.expression, openCalculatedFieldEdit } diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 98d2fbefaf..64917b23c0 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -357,7 +357,7 @@ export class EventTableConfig extends EntityTableConfig { break; case DebugEventType.DEBUG_CALCULATED_FIELD: this.columns[0].width = '80px'; - this.columns[1].width = '20%'; + this.columns[1].width = '100px'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, From a1b9e941e5c11f9022479ea0e0ce52a58041df0c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 18:14:38 +0200 Subject: [PATCH 255/416] fixed debugging dialog height --- .../calculated-field-debug-dialog.component.html | 4 ++-- .../calculated-field-debug-dialog.component.scss | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 8295a9c892..2397162b05 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
        +

        {{ 'calculated-fields.debugging' | translate}}

        @@ -25,7 +25,7 @@ close         -
        +
        Date: Tue, 11 Feb 2025 18:36:26 +0200 Subject: [PATCH 256/416] UI: Fixed apply changes button enabled by default in Device Profile Transport configuration tab --- .../device/device-profile-transport-configuration.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts index 47d782dbdb..a295ef76ab 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts @@ -103,7 +103,7 @@ export class DeviceProfileTransportConfigurationComponent implements ControlValu delete configuration.type; } setTimeout(() => { - this.deviceProfileTransportConfigurationFormGroup.patchValue({configuration}); + this.deviceProfileTransportConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); }, 0); } From 80c747af692a4203d34984903195775f2fe4c4a0 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 18:40:25 +0200 Subject: [PATCH 257/416] amde arguments not required --- .../calculated-field-test-arguments.component.html | 2 +- .../src/app/shared/components/value-input.component.html | 8 ++++---- ui-ngx/src/app/shared/components/value-input.component.ts | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index e6b62a6862..37f6ba4d27 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -28,7 +28,7 @@ - +
        }
        diff --git a/ui-ngx/src/app/shared/components/value-input.component.html b/ui-ngx/src/app/shared/components/value-input.component.html index e6138dd321..a7108f9174 100644 --- a/ui-ngx/src/app/shared/components/value-input.component.html +++ b/ui-ngx/src/app/shared/components/value-input.component.html @@ -32,7 +32,7 @@ - - -
        - Date: Tue, 11 Feb 2025 19:06:05 +0200 Subject: [PATCH 258/416] refactoring --- .../calculated-fields-table-config.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 9560f264b5..ce47292397 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -47,6 +47,7 @@ import { CalculatedFieldScriptTestDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; +import { isObject } from '@core/utils'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -206,7 +207,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig, openCalculatedFieldEdit = true): Observable { - const emptyArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}); - const filledArguments = Object.keys(argumentsObj ?? {}).reduce((acc, key) => { - if (emptyArguments.hasOwnProperty(key)) { - acc[key] = argumentsObj[key]; - } + const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { + acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) ? argumentsObj[key] : ''; return acc; }, {}); return this.dialog.open(CalculatedFieldScriptTestDialogComponent, @@ -273,7 +271,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig Date: Tue, 11 Feb 2025 19:10:55 +0200 Subject: [PATCH 259/416] WIP: arguments --- .../CalculatedFieldTbelScriptEngine.java | 1 + .../ctx/state/ScriptCalculatedFieldState.java | 25 +++++++++- .../api/tbel/DefaultTbelInvokeService.java | 9 ++-- .../thingsboard/script/api/tbel/TbCfArg.java | 22 +++++++++ .../script/api/tbel/TbCfSingleValueArg.java | 30 ++++++++++++ .../script/api/tbel/TbCfTsRollingArg.java | 47 +++++++++++++++++++ 6 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 9e05e05970..6dcbf33067 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import io.jsonwebtoken.lang.Collections; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 4a24d13c93..7d86e376a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -21,12 +21,18 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.mvel2.execution.ExecutionArrayList; +import org.thingsboard.script.api.tbel.TbCfArg; +import org.thingsboard.script.api.tbel.TbCfSingleValueArg; +import org.thingsboard.script.api.tbel.TbCfTsRollingArg; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -62,8 +68,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { } }); Object[] args = ctx.getArgNames().stream() - .map(key -> arguments.get(key).getValue()) + .map(this::toTbelArgument) .toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); return Futures.transform(resultFuture, @@ -72,4 +79,20 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { ); } + private TbCfArg toTbelArgument(String key) { + ArgumentEntry argEntry = arguments.get(key); + if (argEntry instanceof SingleValueArgumentEntry svArg) { + return new TbCfSingleValueArg(svArg.getTs(), argEntry.getValue()); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArg) { + var tsRecords = rollingArg.getTsRecords(); + List values = new ArrayList<>(tsRecords.size()); + for(var e : tsRecords.entrySet()){ + values.add(new TbCfSingleValueArg(e.getKey(), e.getValue().getValue())); + } + return new TbCfTsRollingArg(values); + } else { + throw new RuntimeException("Argument is not supported for TBEL execution!"); + } + } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index 74d6a4e6f5..906e6ac393 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -32,6 +32,7 @@ import org.mvel2.MVEL; import org.mvel2.ParserContext; import org.mvel2.SandboxedParserConfiguration; import org.mvel2.ScriptMemoryOverflowException; +import org.mvel2.integration.PropertyHandlerFactory; import org.mvel2.optimizers.OptimizerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -130,9 +131,11 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem OptimizerFactory.setDefaultOptimizer(OptimizerFactory.SAFE_REFLECTIVE); parserConfig = ParserContext.enableSandboxedMode(); parserConfig.addImport("JSON", TbJson.class); - parserConfig.registerDataType("Date", TbDate.class, date -> 8L); - parserConfig.registerDataType("Random", Random.class, date -> 8L); - parserConfig.registerDataType("Calendar", Calendar.class, date -> 8L); + parserConfig.registerDataType("Date", TbDate.class, val -> 8L); + parserConfig.registerDataType("Random", Random.class, val -> 8L); + parserConfig.registerDataType("Calendar", Calendar.class, val -> 8L); + parserConfig.registerDataType("TbCfSingleValueArg", TbCfSingleValueArg.class, TbCfSingleValueArg::memorySize); + parserConfig.registerDataType("TbCfTsRollingArg", TbCfTsRollingArg.class, TbCfTsRollingArg::memorySize); TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); try { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java new file mode 100644 index 0000000000..f5d990a553 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +public interface TbCfArg { + + long memorySize(); + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java new file mode 100644 index 0000000000..e42fb64ba4 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Data; + +@Data +public class TbCfSingleValueArg implements TbCfArg { + + private final long ts; + private final Object value; + + @Override + public long memorySize() { + return 8L; // TODO; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java new file mode 100644 index 0000000000..a8abbc64ac --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Getter; + +import java.util.List; + +public class TbCfTsRollingArg implements TbCfArg { + + @Getter + private final List values; + + public TbCfTsRollingArg(List values) { + this.values = values; + } + + @Override + public long memorySize() { + return values.size() * 8L; //TODO; + } + + public double max() { + double max = Double.MIN_VALUE; + for (TbCfSingleValueArg arg : values) { + double val = Double.valueOf(arg.getValue().toString()); + if (max < val) { + max = val; + } + } + return max; + } + +} From 79f856bdca8de91a8ba19cce7289cc1fc1eccf64 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 11 Feb 2025 18:10:49 +0100 Subject: [PATCH 260/416] fixed 'value too long' for saving TbResource audit logs --- .../server/service/resource/DefaultTbResourceService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java index e3a79a8ba5..5ff589fd2d 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java @@ -80,7 +80,7 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements resource.setResourceKey(resource.getFileName()); } TbResource savedResource = resourceService.saveResource(resource); - logEntityActionService.logEntityAction(tenantId, savedResource.getId(), savedResource, actionType, user); + logEntityActionService.logEntityAction(tenantId, savedResource.getId(), new TbResourceInfo(savedResource), actionType, user); return savedResource; } catch (Exception e) { logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.TB_RESOURCE), From e542267295c2818eb87f5f17b40deb16097b4f66 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 15:57:12 +0200 Subject: [PATCH 261/416] Added calculated field responsive styles --- ...lated-field-arguments-table.component.html | 18 ++--- ...lated-field-arguments-table.component.scss | 6 ++ ...lculated-field-debug-dialog.component.html | 4 +- ...lculated-field-debug-dialog.component.scss | 10 ++- .../calculated-field-dialog.component.html | 2 +- ...ulated-field-test-arguments.component.html | 4 +- ...ulated-field-test-arguments.component.scss | 29 ++++++++ ...lculated-field-test-arguments.component.ts | 1 + ...ed-field-script-test-dialog.component.html | 6 +- ...ated-field-script-test-dialog.component.ts | 73 ++++++++++++++----- .../components/event/event-table-config.ts | 8 +- 11 files changed, 120 insertions(+), 41 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 5ca80b8214..796fe7c138 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -18,19 +18,19 @@
        -
        {{ 'calculated-fields.argument-name' | translate }}
        -
        {{ 'calculated-fields.datasource' | translate }}
        -
        {{ 'common.type' | translate }}
        -
        {{ 'entity.key' | translate }}
        +
        {{ 'calculated-fields.argument-name' | translate }}
        +
        {{ 'calculated-fields.datasource' | translate }}
        +
        {{ 'common.type' | translate }}
        +
        {{ 'entity.key' | translate }}
        @for (group of argumentsFormArray.controls; track group) {
        - + -
        +
        @if (group.get('refEntityId')?.get('id')?.value) { @@ -63,7 +63,7 @@ }
        - + @if (group.get('refEntityKey').get('type').value; as type) { @@ -72,9 +72,9 @@ } - + -
        +
        {{ group.get('refEntityKey').get('key').value }}
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index a0c90bba32..e6a48dee58 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -27,4 +27,10 @@ font-size: 14px; } } + + .mat-mdc-standard-chip { + .mdc-evolution-chip__cell--primary, .mdc-evolution-chip__action--primary, .mdc-evolution-chip__text-label { + overflow: hidden; + } + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 2397162b05..7da70e180e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
        +

        {{ 'calculated-fields.debugging' | translate}}

        @@ -25,7 +25,7 @@ close         -
        +
        -
        +

        {{ 'entity.type-calculated-field' | translate}}

        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index 37f6ba4d27..03781f1ded 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -19,7 +19,7 @@
        {{ 'calculated-fields.arguments' | translate }}
        -
        {{ 'calculated-fields.argument-name' | translate }}
        +
        {{ 'calculated-fields.argument-name' | translate }}
        {{ 'common.value' | translate }}
        @@ -28,7 +28,7 @@ - +
        }
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss new file mode 100644 index 0000000000..1b2c8670c1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host::ng-deep { + .tb-form-table-row { + .argument-value { + .tb-value-type.row { + @media #{constants.$mat-lt-sm} { + width: 100px; + min-width: 100px; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts index d7a33d1a89..c8c9f4e778 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts @@ -30,6 +30,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @Component({ selector: 'tb-calculated-field-test-arguments', templateUrl: './calculated-field-test-arguments.component.html', + styleUrls: ['./calculated-field-test-arguments.component.scss'], providers: [ { provide: NG_VALUE_ACCESSOR, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html index 03582b3066..3d3fe79e4b 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
        +

        {{ 'calculated-fields.test-script-function' | translate }} ({{ 'api-usage.tbel' | translate }})

         -
        +
        {{ 'common.general' | translate }}
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 4e9819a786..8b3bb899ad 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
        +
        {{ 'calculated-fields.argument-settings' | translate }}
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..cfe7ee9461 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host { + .fixed-title-width { + @media #{constants.$mat-lt-sm} { + min-width: 120px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index a2e5545926..66ce0aeeeb 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -42,6 +42,7 @@ import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] }) export class CalculatedFieldArgumentPanelComponent implements OnInit { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index 03781f1ded..08d023591a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -28,7 +28,7 @@ - +
        }
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss index 2420aa5a50..eaee8e443d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -26,6 +26,7 @@ padding-top: 5px; padding-left: 5px; border: 1px solid #c0c0c0; + overflow: scroll; } .block-label-container { From 8b3fc83acc140ea33e415d741ba5c7fe5f9aed6c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 18:07:48 +0200 Subject: [PATCH 266/416] style fixes --- .../calculated-field-dialog.component.html | 2 +- .../calculated-field-dialog.component.scss | 29 +++++++++++++++++++ .../calculated-field-dialog.component.ts | 1 + ...ulated-field-argument-panel.component.html | 2 +- ...ulated-field-argument-panel.component.scss | 7 +++++ 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 20245414dd..6ff0977178 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
        +

        {{ 'entity.type-calculated-field' | translate}}

        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss new file mode 100644 index 0000000000..300193a6c1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host { + .dialog-container { + + @media #{constants.$mat-sm} { + min-width: 526px; + } + + @media #{constants.$mat-gt-sm} { + min-width: 770px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index e75c962268..31f07a1c0d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -40,6 +40,7 @@ import { ScriptLanguage } from '@shared/models/rule-node.models'; @Component({ selector: 'tb-calculated-field-dialog', templateUrl: './calculated-field-dialog.component.html', + styleUrls: ['./calculated-field-dialog.component.scss'], }) export class CalculatedFieldDialogComponent extends DialogComponent implements AfterViewInit { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 8b3bb899ad..c2fe831204 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
        +
        {{ 'calculated-fields.argument-settings' | translate }}
        diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss index cfe7ee9461..7c4e5c2f5e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -16,6 +16,13 @@ @use '../../../../../../../scss/constants' as constants; :host { + display: flex; + min-width: 520px; + + @media #{constants.$mat-lt-sm} { + min-width: 320px; + } + .fixed-title-width { @media #{constants.$mat-lt-sm} { min-width: 120px; From 9162acedee26f7ade43a6ad4dce9133de19b481b Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 18:19:06 +0200 Subject: [PATCH 267/416] style fixes --- .../panel/calculated-field-argument-panel.component.scss | 2 +- .../calculated-field-test-arguments.component.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss index 7c4e5c2f5e..ca6d773262 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -20,7 +20,7 @@ min-width: 520px; @media #{constants.$mat-lt-sm} { - min-width: 320px; + min-width: 280px; } .fixed-title-width { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index 08d023591a..eea3523d00 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -28,7 +28,7 @@ - +
        }
        From 1f28c0efe9b8a4f1d0c6f50d0c790dea6c2cfc27 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 18:40:41 +0200 Subject: [PATCH 268/416] style fixes --- .../calculated-field-arguments-table.component.html | 2 +- .../dialog/calculated-field-dialog.component.scss | 10 ++-------- .../calculated-field-argument-panel.component.scss | 7 ++----- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 796fe7c138..77347c6477 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -80,7 +80,7 @@ -
        +
        @for (group of argumentsFormArray.controls; track group) { @@ -80,7 +80,7 @@ -
        +
        -
        +
        Date: Thu, 13 Feb 2025 12:33:30 +0200 Subject: [PATCH 274/416] fixed entity/message id min width --- .../app/modules/home/components/event/event-table-config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 2f2650d434..b32c5f0050 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -359,7 +359,7 @@ export class EventTableConfig extends EntityTableConfig { this.columns[0].width = '80px'; this.columns[1].width = '100px'; this.columns.push( - new EntityTableColumn('entityId', 'event.entity-id', '85px', + new EntityTableColumn('entityId', 'event.entity-id', '100px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, @@ -379,7 +379,7 @@ export class EventTableConfig extends EntityTableConfig { type: CellActionDescriptorType.COPY_BUTTON } ), - new EntityTableColumn('messageId', 'event.message-id', '85px', + new EntityTableColumn('messageId', 'event.message-id', '100px', (entity) => entity.body.msgId ? `${entity.body.msgId?.substring(0, 8)}…` : '-', () => ({padding: '0 12px 0 0'}), false, From 7232d939d86bbfe368cf955f08cd980efe3030f1 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Feb 2025 12:57:26 +0200 Subject: [PATCH 275/416] fixed cf version --- .../server/dao/model/sql/CalculatedFieldEntity.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 64c8e8d5b8..55928dbed1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.BaseVersionedEntity; import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; @@ -52,7 +52,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.DEBUG_SETTINGS; @EqualsAndHashCode(callSuper = true) @Entity @Table(name = CALCULATED_FIELD_TABLE_NAME) -public class CalculatedFieldEntity extends BaseSqlEntity implements BaseEntity { +public class CalculatedFieldEntity extends BaseVersionedEntity implements BaseEntity { @Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN) private UUID tenantId; From 677385e8babc8f263e2b662c6142cd6311d40f4c Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 13 Feb 2025 13:01:17 +0200 Subject: [PATCH 276/416] CF states restore from Kafka --- ...CalculatedFieldEntityMessageProcessor.java | 9 +- .../AbstractCalculatedFieldStateService.java | 173 +++++++++++++++++ .../cf/CalculatedFieldStateService.java | 5 + .../cf/CfRocksDb.java} | 30 +-- ...faultCalculatedFieldProcessingService.java | 4 +- .../server/service/cf/RocksDBService.java | 97 ---------- ...aultCalculatedFieldEntityProfileCache.java | 11 +- .../cf/ctx/CalculatedFieldEntityCtxId.java | 5 + .../KafkaCalculatedFieldStateService.java | 129 ++++++++++++- .../RocksDBCalculatedFieldStateService.java | 175 +++--------------- ...faultTbCalculatedFieldConsumerService.java | 46 +++-- .../queue/DefaultTbClusterService.java | 9 +- .../DefaultTbRuleEngineConsumerService.java | 5 +- .../consumer/MainQueueConsumerManager.java | 10 +- .../queue/ruleengine/TbQueueConsumerTask.java | 12 +- .../auth/rest/RestAuthenticationProvider.java | 1 + .../thingsboard/server/utils/TbRocksDb.java | 71 +++++++ .../src/main/resources/thingsboard.yml | 8 +- .../server/common/data/DataConstants.java | 1 + .../common/data/util/CollectionsUtil.java | 12 ++ common/proto/src/main/proto/queue.proto | 13 +- .../AbstractTbQueueConsumerTemplate.java | 6 +- .../queue/discovery/HashPartitionService.java | 54 ++++-- .../queue/discovery/PartitionService.java | 2 + .../server/queue/discovery/QueueKey.java | 6 + .../discovery/event/PartitionChangeEvent.java | 6 +- .../server/queue/kafka/KafkaTbQueueMsg.java | 9 +- .../queue/kafka/TbKafkaConsumerTemplate.java | 63 ++++++- .../queue/kafka/TbKafkaProducerTemplate.java | 7 +- .../queue/kafka/TbKafkaTopicConfigs.java | 5 + .../InMemoryMonolithQueueFactory.java | 5 +- .../provider/KafkaMonolithQueueFactory.java | 40 ++-- .../KafkaTbRuleEngineQueueFactory.java | 40 ++-- .../provider/TbRuleEngineQueueFactory.java | 6 +- .../server/queue/util/AfterStartUp.java | 9 +- 35 files changed, 679 insertions(+), 405 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java rename application/src/main/java/org/thingsboard/server/{utils/RocksDBConfig.java => service/cf/CfRocksDb.java} (58%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 61f7abb297..c45c4c2ef0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -96,8 +96,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldStateRestoreMsg msg) { - log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), msg.getId().cfId()); - states.put(msg.getId().cfId(), msg.getState()); + CalculatedFieldId cfId = msg.getId().cfId(); + log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); + if (msg.getState() != null) { + states.put(cfId, msg.getState()); + } else { + states.remove(cfId); + } } public void process(EntityInitCalculatedFieldMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java new file mode 100644 index 0000000000..7df20bc139 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -0,0 +1,173 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; + +import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; + +public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { + + @Autowired + private ActorSystemContext actorSystemContext; + + @Override + public final void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + CalculatedFieldStateMsgProto stateMsg = toProto(stateId, state); + long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); + if (maxStateSizeInKBytes <= 0 || stateMsg.getSerializedSize() <= maxStateSizeInKBytes) { + doPersist(stateId, stateMsg, callback); + } + } + + protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback); + + @Override + public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + doRemove(stateId, callback); + } + + protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); + + protected void processRestoredState(CalculatedFieldStateMsgProto stateMsg) { + CalculatedFieldEntityCtxId stateId = fromProto(stateMsg.getId()); + CalculatedFieldState state = stateMsg.hasState() ? fromProto(stateMsg.getState()) : null; + actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(stateId, state)); + } + + protected CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + protected CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { + TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + protected CalculatedFieldStateMsgProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { + var stateProto = CalculatedFieldStateProto.newBuilder() + .setType(state.getType().name()); + state.getArguments().forEach((argName, argEntry) -> { + if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + stateProto.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { + stateProto.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); + } + }); + return CalculatedFieldStateMsgProto.newBuilder() + .setId(toProto(stateId)) + .setState(stateProto) + .build(); + } + + protected TransportProtos.SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { + TransportProtos.SingleValueArgumentProto.Builder builder = TransportProtos.SingleValueArgumentProto.newBuilder() + .setArgName(argName); + if (entry != SingleValueArgumentEntry.EMPTY) { + builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + } + Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); + return builder.build(); + } + + protected TransportProtos.TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { + TransportProtos.TsValueListProto.Builder builder = TransportProtos.TsValueListProto.newBuilder().setKey(argName); + if (entry != TsRollingArgumentEntry.EMPTY) { + entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); + } + return builder.build(); + } + + protected CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { + if (StringUtils.isEmpty(proto.getType())) { + return null; + } + + CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); + + CalculatedFieldState state = switch (type) { + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); + }; + + proto.getSingleValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); + + if (CalculatedFieldType.SCRIPT.equals(type)) { + proto.getRollingValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); + } + + return state; + } + + protected SingleValueArgumentEntry fromSingleValueArgumentProto(TransportProtos.SingleValueArgumentProto proto) { + if (!proto.hasValue()) { + return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY; + } + TransportProtos.TsValueProto tsValueProto = proto.getValue(); + long ts = tsValueProto.getTs(); + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); + return new SingleValueArgumentEntry(ts, kvEntry, proto.getVersion()); + } + + protected TsRollingArgumentEntry fromRollingArgumentProto(TransportProtos.TsValueListProto proto) { + if (proto.getTsValueCount() <= 0) { + return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY; + } + TreeMap tsRecords = new TreeMap<>(); + proto.getTsValueList().forEach(tsValueProto -> { + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); + tsRecords.put(tsValueProto.getTs(), kvEntry); + }); + return new TsRollingArgumentEntry(tsRecords); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index 37211c66c5..05f9dab36b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -16,14 +16,19 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import java.util.Set; + public interface CalculatedFieldStateService { void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); + void restore(Set partitions); + } diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java rename to application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java index 9c3c02f472..cc63d757a3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java @@ -13,40 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.service.cf; import jakarta.annotation.PreDestroy; import org.rocksdb.Options; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; +import org.rocksdb.WriteOptions; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.thingsboard.server.utils.TbRocksDb; @Component -public class RocksDBConfig { +public class CfRocksDb extends TbRocksDb { - @Value("${rocksdb.db_path:${java.io.tmpdir}/rocksdb}") - private String dbPath; - private RocksDB db; - - static { - RocksDB.loadLibrary(); - } - - public RocksDB getDb() throws RocksDBException { - if (db == null) { - Options options = new Options().setCreateIfMissing(true); - db = RocksDB.open(options, dbPath); - } - return db; + public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) throws Exception { + super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); } @PreDestroy + @Override public void close() { - if (db != null) { - db.close(); - db = null; - } + super.close(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 6b47053f2d..f2c6976cbf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -70,6 +70,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; @@ -91,7 +92,6 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @TbRuleEngineComponent @Service @@ -188,7 +188,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP if (broadcast) { broadcasts.add(link); } else { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, link.entityId()); + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, link.entityId()); unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java deleted file mode 100644 index 7181cc43ed..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf; - -import lombok.extern.slf4j.Slf4j; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; -import org.rocksdb.WriteOptions; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.utils.RocksDBConfig; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -@Service -@Slf4j -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) -public class RocksDBService { - - private final RocksDB db; - private final WriteOptions writeOptions; - - public RocksDBService(RocksDBConfig config) throws RocksDBException { - this.db = config.getDb(); - this.writeOptions = new WriteOptions().setSync(true); - } - - public void put(String key, String value) { - try { - db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8)); - } catch (RocksDBException e) { - log.error("Failed to store data to RocksDB", e); - } - } - - public void put(CalculatedFieldEntityCtxIdProto key, CalculatedFieldStateProto value) { - try { - db.put(writeOptions, key.toByteArray(), value.toByteArray()); - } catch (RocksDBException e) { - log.error("Failed to store data to RocksDB", e); - } - } - - public void delete(CalculatedFieldEntityCtxIdProto key) { - try { - db.delete(writeOptions, key.toByteArray()); - } catch (RocksDBException e) { - log.error("Failed to delete data from RocksDB", e); - } - } - - public String get(String key) { - try { - byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); - return value != null ? new String(value, StandardCharsets.UTF_8) : null; - } catch (RocksDBException e) { - log.error("Failed to retrieve data from RocksDB", e); - return null; - } - } - - public Map getAll() { - Map results = new HashMap<>(); - try (RocksIterator iterator = db.newIterator()) { - for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - try { - CalculatedFieldEntityCtxIdProto key = CalculatedFieldEntityCtxIdProto.parseFrom(iterator.key()); - CalculatedFieldStateProto value = CalculatedFieldStateProto.parseFrom(iterator.value()); - results.put(key, value); - } catch (Exception e) { - log.error("Failed to retrieve data from RocksDB", e); - } - } - } - return results; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 9a56d35ac4..b2bb2287fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -21,8 +21,8 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -30,7 +30,6 @@ import org.thingsboard.server.queue.util.TbRuleEngineComponent; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; @@ -49,7 +48,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - myPartitions = event.getCalculatedFieldsPartitions().stream() + myPartitions = event.getCfPartitions().stream() .filter(TopicPartitionInfo::isMyPartition) .map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); //Naive approach that need to be improved. @@ -58,7 +57,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { - var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var tpi = partitionService.resolve(QueueKey.CF, entityId); var partition = tpi.getPartition().orElse(UNKNOWN); tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) .add(profileId, entityId, partition, tpi.isMyPartition()); @@ -66,7 +65,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { - var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var tpi = partitionService.resolve(QueueKey.CF, entityId); var partition = tpi.getPartition().orElse(UNKNOWN); var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); //TODO: make this method atomic; @@ -87,7 +86,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { - var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var tpi = partitionService.resolve(QueueKey.CF, entityId); return tpi.getPartition().orElse(UNKNOWN); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index 6ce0a11ade..a09a5cd035 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -20,4 +20,9 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { + + public String toKey() { + return cfId + "_" + entityId; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index a1999d438c..152b1affd9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -15,31 +15,140 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.queue.DefaultTbCalculatedFieldConsumerService.CalculatedFieldQueueConfig; +import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; + +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${zk.enabled:false}'=='true' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") -public class KafkaCalculatedFieldStateService implements CalculatedFieldStateService { +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldStateService { + + private final TbRuleEngineQueueFactory queueFactory; + private final PartitionService partitionService; + + @Value("${queue.calculated_fields.poll_interval:25}") + private long pollInterval; + @Value("${queue.calculated_fields.consumer_per_partition:true}") + private boolean consumerPerPartition; + + private MainQueueConsumerManager, CalculatedFieldQueueConfig> stateConsumer; + private TbKafkaProducerTemplate> stateProducer; + + protected ExecutorService consumersExecutor; + protected ExecutorService mgmtExecutor; + protected ScheduledExecutorService scheduler; - @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) - public void initCalculatedFieldStates() { + private final AtomicInteger counter = new AtomicInteger(); + + @PostConstruct + private void init() { + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("cf-state-consumer")); + this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(Math.max(Runtime.getRuntime().availableProcessors(), 4), "cf-state-mgmt"); + this.scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("cf-state-consumer-scheduler"); + + this.stateConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() + .queueKey(QueueKey.CF_STATES) + .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg msg : msgs) { + try { + processRestoredState(msg.getValue()); + } catch (Throwable t) { + log.error("Failed to process state message: {}", msg, t); + } + + int processedMsgCount = counter.incrementAndGet(); + if (processedMsgCount % 10000 == 0) { + log.info("Processed {} calculated field state msgs", processedMsgCount); + } + } + }) + .consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) + .consumerExecutor(consumersExecutor) + .scheduler(scheduler) + .taskExecutor(mgmtExecutor) + .build(); + this.stateProducer = (TbKafkaProducerTemplate>) queueFactory.createCalculatedFieldStateProducer(); } @Override - public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - callback.onSuccess(); + protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF_STATES, stateId.entityId()); + stateProducer.send(tpi, stateId.toKey(), new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto), new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (callback != null) { + callback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }); } @Override - public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { - callback.onSuccess(); + protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + doPersist(stateId, CalculatedFieldStateMsgProto.newBuilder() + .setId(toProto(stateId)) + .build(), callback); + } + + @Override + public void restore(Set partitions) { + partitions = partitions.stream().map(tpi -> tpi.newByTopic(partitionService.getTopic(QueueKey.CF_STATES))).collect(Collectors.toSet()); + log.info("Restoring calculated field states for partitions: {}", partitions.stream().map(TopicPartitionInfo::getFullTopicName).toList()); + long startTs = System.currentTimeMillis(); + counter.set(0); + + stateConsumer.doUpdate(partitions); // calling blocking doUpdate instead of update + stateConsumer.awaitStop(0);// consumers should stop on their own because stopWhenRead is true, we just need to wait + + log.info("Restored {} calculated field states in {} ms", counter.get(), System.currentTimeMillis() - startTs); + } + + @PreDestroy + private void preDestroy() { + stateConsumer.stop(); + stateConsumer.awaitStop(); + stateProducer.stop(); + + consumersExecutor.shutdownNow(); + mgmtExecutor.shutdownNow(); + scheduler.shutdownNow(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 3374eb2660..eb7f4818d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -15,174 +15,57 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.RequiredArgsConstructor; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.KvProtoUtil; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; -import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.service.cf.RocksDBService; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; +import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; +import org.thingsboard.server.service.cf.CfRocksDb; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.CalculatedFieldStateService; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.UUID; -import java.util.stream.Collectors; +import java.util.Set; @Service @RequiredArgsConstructor -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) // Queue type in mem or Kafka; -public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateService { +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") +public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldStateService { - private final ActorSystemContext actorSystemContext; - private final RocksDBService rocksDBService; + private final CfRocksDb cfRocksDb; - public Map restoreStates() { - return rocksDBService.getAll().entrySet().stream() - .collect(Collectors.toMap( - entry -> fromProto(entry.getKey()), - entry -> fromProto(entry.getValue()) - )); - } - - @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) - public void initCalculatedFieldStates() { - restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); - } + private Set partitions; @Override - public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - CalculatedFieldStateProto stateProto = toProto(stateId, state); - long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); - if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= maxStateSizeInKBytes) { - rocksDBService.put(toProto(stateId), stateProto); - } + protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback) { + cfRocksDb.put(stateId.toKey(), stateMsgProto.toByteArray()); callback.onSuccess(); } @Override - public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { - rocksDBService.delete(toProto(ctxId)); + protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + cfRocksDb.delete(stateId.toKey()); callback.onSuccess(); } - private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return CalculatedFieldEntityCtxIdProto.newBuilder() - .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - - private CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { - TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); - } - - private CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { - CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() - .setId(toProto(stateId)) - .setType(state.getType().name()); - - state.getArguments().forEach((argName, argEntry) -> { - if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); - } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { - builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); - } - }); - - return builder.build(); - } - - private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { - SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() - .setArgName(argName); - - if (entry != SingleValueArgumentEntry.EMPTY) { - builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); - } - - Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); - - return builder.build(); - } - - private TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { - TsValueListProto.Builder builder = TsValueListProto.newBuilder().setKey(argName); - - if (entry != TsRollingArgumentEntry.EMPTY) { - entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); - } - - return builder.build(); - } - - private CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { - if (StringUtils.isEmpty(proto.getType())) { - return null; - } - - CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); - - CalculatedFieldState state = switch (type) { - case SIMPLE -> new SimpleCalculatedFieldState(); - case SCRIPT -> new ScriptCalculatedFieldState(); - }; - - proto.getSingleValueArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); - - if (CalculatedFieldType.SCRIPT.equals(type)) { - proto.getRollingValueArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); - } - - return state; - } - - private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { - if (!proto.hasValue()) { - return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY; + @Override + public void restore(Set partitions) { + if (this.partitions == null) { + this.partitions = partitions; + } else { + return; } - TsValueProto tsValueProto = proto.getValue(); - long ts = tsValueProto.getTs(); - BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); - return new SingleValueArgumentEntry(ts, kvEntry, proto.getVersion()); - } - private TsRollingArgumentEntry fromRollingArgumentProto(TsValueListProto proto) { - if (proto.getTsValueCount() <= 0) { - return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY; - } - TreeMap tsRecords = new TreeMap<>(); - proto.getTsValueList().forEach(tsValueProto -> { - BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); - tsRecords.put(tsValueProto.getTs(), kvEntry); + cfRocksDb.forEach((key, value) -> { + try { + processRestoredState(CalculatedFieldStateMsgProto.parseFrom(value)); + } catch (InvalidProtocolBufferException e) { + log.error("[{}] Failed to process restored state", key, e); + } }); - return new TsRollingArgumentEntry(tsRecords); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index f3e976084b..82ee4de7dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,11 +21,11 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; @@ -52,6 +52,7 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; @@ -65,6 +66,8 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -86,10 +89,12 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer private int poolSize; private final TbRuleEngineQueueFactory queueFactory; + private final CalculatedFieldStateService stateService; private MainQueueConsumerManager, CalculatedFieldQueueConfig> mainConsumer; - private volatile ListeningExecutorService calculatedFieldsExecutor; + private ListeningExecutorService calculatedFieldsExecutor; + private ExecutorService repartitionExecutor; public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, @@ -100,19 +105,22 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer PartitionService partitionService, ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, - CalculatedFieldCache calculatedFieldCache) { + CalculatedFieldCache calculatedFieldCache, + CalculatedFieldStateService stateService) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; + this.stateService = stateService; } @PostConstruct public void init() { super.init("tb-cf"); this.calculatedFieldsExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(poolSize, "tb-cf-executor")); // TODO: multiple threads. + this.repartitionExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-cf-repartition")); this.mainConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() - .queueKey(new QueueKey(ServiceType.TB_RULE_ENGINE)) + .queueKey(QueueKey.CF) .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) @@ -137,20 +145,23 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - var partitions = event.getCalculatedFieldsPartitions(); - log.info("Subscribing to partitions: {}", partitions); - // TODO: @vklimov - before update of the main consumer, we should read the state topics and use - // CalculatedFieldStateService (KafkaCalculatedFieldStateService) to restore the states for entities that belong to new partitions. - // Cleanup entities that do not belong to current partition; - mainConsumer.update(event.getCalculatedFieldsPartitions()); - // Cleanup old entities after corresponding consumers are stopped. - // Any periodic tasks need to check that the entity is still managed by the current server before processing. - actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + var partitions = event.getCfPartitions(); + repartitionExecutor.submit(() -> { + try { + stateService.restore(partitions); + mainConsumer.update(partitions); + // Cleanup old entities after corresponding consumers are stopped. + // Any periodic tasks need to check that the entity is still managed by the current server before processing. + actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + } catch (Throwable t) { + log.error("Failed to process partition change event: {}", event, t); + } + }); } private boolean[] partitionsToBooleanIndexArray(Set partitions) { boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; - for(var tpi : partitions) { + for (var tpi : partitions) { tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); } return myPartitions; @@ -193,9 +204,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer packSubmitFuture.cancel(true); log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); } - if (log.isDebugEnabled()) { - ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); - } +// if (log.isDebugEnabled()) { +// ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); +// } + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); // TODO: replace with commented above after testing ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 48d26895b9..920a7563dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -94,6 +94,7 @@ import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; @@ -111,7 +112,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static org.thingsboard.server.common.util.ProtoUtils.toProto; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -358,7 +358,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, entityId); pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, callback); } @@ -371,7 +371,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, entityId); producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); toRuleEngineNfs.incrementAndGet(); } @@ -792,8 +792,7 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> - pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 5e07e33a9e..c42f7c490b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rpc.RpcError; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -63,8 +64,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; - @Service @TbRuleEngineComponent @Slf4j @@ -109,7 +108,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { event.getPartitionsMap().forEach((queueKey, partitions) -> { - if (CALCULATED_FIELD_QUEUE_KEY.equals(queueKey)) { + if (CollectionsUtil.isOneOf(queueKey, QueueKey.CF, QueueKey.CF_STATES)) { return; } if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java index 6eb5c94c9b..5fa69695d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java @@ -182,7 +182,7 @@ public class MainQueueConsumerManager partitions) { + public void doUpdate(Set partitions) { this.partitions = partitions; consumerWrapper.updatePartitions(partitions); } @@ -226,7 +226,9 @@ public class MainQueueConsumerManager msgs, TbQueueConsumer consumer, C config) throws Exception { + log.trace("Processing {} messages", msgs.size()); msgPackProcessor.process(msgs, consumer, config); + log.trace("Processed {} messages", msgs.size()); } public void stop() { @@ -236,8 +238,12 @@ public class MainQueueConsumerManager consumerTask.awaitCompletion(timeoutSec)); log.debug("[{}] Unsubscribed and stopped consumers", queueKey); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index 5e672eb5c6..0b4e7c02d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -70,13 +70,21 @@ public class TbQueueConsumerTask { } public void awaitCompletion() { + awaitCompletion(30); + } + + public void awaitCompletion(long timeoutSec) { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { - task.get(30, TimeUnit.SECONDS); + if (timeoutSec > 0) { + task.get(timeoutSec, TimeUnit.SECONDS); + } else { + task.get(); + } log.trace("[{}] Awaited finish", key); } catch (Exception e) { - log.warn("[{}] Failed to await for consumer to stop", key, e); + log.warn("[{}] Failed to await for consumer to stop (timeout {} sec)", key, timeoutSec, e); } task = null; } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index b9fe54deec..56215b82ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -106,6 +106,7 @@ public class RestAuthenticationProvider implements AuthenticationProvider { if (twoFactorAuthService.isTwoFaEnabled(securityUser.getTenantId(), securityUser.getId())) { return new MfaAuthenticationToken(securityUser); } else { + systemSecurityService.logLoginAction(securityUser, authentication.getDetails(), ActionType.LOGIN, null); } } else { diff --git a/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java b/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java new file mode 100644 index 0000000000..fa56e0ec1d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import lombok.SneakyThrows; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteOptions; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.BiConsumer; + +public class TbRocksDb { + + protected final String path; + private final WriteOptions writeOptions; + protected final RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + public TbRocksDb(String path, Options dbOptions, WriteOptions writeOptions) throws Exception { + this.path = path; + this.writeOptions = writeOptions; + Files.createDirectories(Path.of(path).getParent()); + this.db = RocksDB.open(dbOptions, path); + } + + @SneakyThrows + public void put(String key, byte[] value) { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value); + } + + public void forEach(BiConsumer processor) { + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + processor.accept(key, iterator.value()); + } + } + } + + @SneakyThrows + public void delete(String key) { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + } + + public void close() { + if (db != null) { + db.close(); + } + } + +} \ No newline at end of file diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 197f05a5d5..e2570e9137 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -426,10 +426,6 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls -rocksdb: - # Rocksdb path - db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" - # Actor system parameters actors: system: @@ -1629,6 +1625,8 @@ queue: edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Calculated Field topics calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Calculated Field State topics + calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1763,6 +1761,8 @@ queue: consumer_per_partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" # Thread pool size for processing of the incoming messages pool_size: "${TB_QUEUE_CF_POOL_SIZE:8}" + # RocksDB path for storing CF states + rocks_db_path: "${TB_QUEUE_CF_ROCKS_DB_PATH:${user.home}/.rocksdb/cf_states}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 77a7c4a781..c84cae40dd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -146,5 +146,6 @@ public class DataConstants { public static final String EDGE_EVENT_QUEUE_NAME = "EdgeEvent"; public static final String CF_QUEUE_NAME = "CalculatedFields"; + public static final String CF_STATES_QUEUE_NAME = "CalculatedFieldStates"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index e89f3c1ad8..7f9a611c98 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -75,4 +75,16 @@ public class CollectionsUtil { return isEmpty(collection) || collection.contains(element); } + public static boolean isOneOf(V value, V... others) { + if (value == null) { + return false; + } + for (V other : others) { + if (value.equals(other)) { + return true; + } + } + return false; + } + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4a2f3710ae..992c6ac42f 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -814,12 +814,15 @@ message SingleValueArgumentProto { int64 version = 3; } -message CalculatedFieldStateProto { +message CalculatedFieldStateMsgProto { CalculatedFieldEntityCtxIdProto id = 1; - // int32 version = 2; - string type = 3; - repeated SingleValueArgumentProto singleValueArguments = 4; - repeated TsValueListProto rollingValueArguments = 5; + CalculatedFieldStateProto state = 2; +} + +message CalculatedFieldStateProto { + string type = 1; + repeated SingleValueArgumentProto singleValueArguments = 2; + repeated TsValueListProto rollingValueArguments = 3; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 9513565ca1..6a5b86cf54 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -120,9 +120,9 @@ public abstract class AbstractTbQueueConsumerTemplate i if (record != null) { result.add(decode(record)); } - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - throw new RuntimeException("Failed to decode record: ", e); + } catch (Exception e) { + log.error("Failed to decode record {}", record, e); + throw new RuntimeException("Failed to decode record " + record, e); } }); return result; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index f2f1ccd19c..249dfa859f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -51,7 +51,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.DataConstants.*; +import static org.thingsboard.server.common.data.DataConstants.EDGE_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; @Service @Slf4j @@ -78,8 +79,6 @@ public class HashPartitionService implements PartitionService { @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; - public static final QueueKey CALCULATED_FIELD_QUEUE_KEY = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); - private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; private final TenantRoutingInfoService tenantRoutingInfoService; @@ -120,8 +119,10 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); - partitionSizesMap.put(CALCULATED_FIELD_QUEUE_KEY, cfPartitions); - partitionTopicsMap.put(CALCULATED_FIELD_QUEUE_KEY, cfEventTopic); + partitionSizesMap.put(QueueKey.CF, cfPartitions); + partitionTopicsMap.put(QueueKey.CF, cfEventTopic); + partitionSizesMap.put(QueueKey.CF_STATES, cfPartitions); + partitionTopicsMap.put(QueueKey.CF_STATES, cfStateTopic); QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); @@ -148,6 +149,11 @@ public class HashPartitionService implements PartitionService { return myPartitions.get(queueKey); } + @Override + public String getTopic(QueueKey queueKey) { + return partitionTopicsMap.get(queueKey); + } + private void doInitRuleEnginePartitions() { List queueRoutingInfoList = getQueueRoutingInfos(); queueRoutingInfoList.forEach(queue -> { @@ -222,7 +228,7 @@ public class HashPartitionService implements PartitionService { }); if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { publishPartitionChangeEvent(ServiceType.TB_RULE_ENGINE, queueKeys.stream() - .collect(Collectors.toMap(k -> k, k -> Collections.emptySet()))); + .collect(Collectors.toMap(k -> k, k -> Collections.emptySet())), Collections.emptyMap()); } } @@ -402,6 +408,7 @@ public class HashPartitionService implements PartitionService { myPartitions = newPartitions; Map> changedPartitionsMap = new HashMap<>(); + Map> oldPartitionsMap = new HashMap<>(); Set removed = new HashSet<>(); oldPartitions.forEach((queueKey, partitions) -> { @@ -422,19 +429,16 @@ public class HashPartitionService implements PartitionService { myPartitions.forEach((queueKey, partitions) -> { if (!partitions.equals(oldPartitions.get(queueKey))) { - Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(queueKey, partition)) - .collect(Collectors.toSet()); - changedPartitionsMap.put(queueKey, tpiList); + changedPartitionsMap.put(queueKey, toTpiList(queueKey, partitions)); + oldPartitionsMap.put(queueKey, toTpiList(queueKey, oldPartitions.get(queueKey))); } }); if (!changedPartitionsMap.isEmpty()) { - Map>> partitionsByServiceType = new HashMap<>(); - changedPartitionsMap.forEach((queueKey, partitions) -> { - partitionsByServiceType.computeIfAbsent(queueKey.getType(), serviceType -> new HashMap<>()) - .put(queueKey, partitions); - }); - partitionsByServiceType.forEach(this::publishPartitionChangeEvent); + changedPartitionsMap.entrySet().stream() + .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .forEach((serviceType, partitionsMap) -> { + publishPartitionChangeEvent(serviceType, partitionsMap, oldPartitionsMap); + }); } if (currentOtherServices == null) { @@ -466,13 +470,15 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } - private void publishPartitionChangeEvent(ServiceType serviceType, Map> partitionsMap) { - log.info("Partitions changed: {}", System.lineSeparator() + partitionsMap.entrySet().stream() + private void publishPartitionChangeEvent(ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { + log.info("Partitions changed: {}", System.lineSeparator() + newPartitions.entrySet().stream() .map(entry -> "[" + entry.getKey() + "] - [" + entry.getValue().stream() .map(tpi -> tpi.getPartition().orElse(-1).toString()).sorted() .collect(Collectors.joining(", ")) + "]") .collect(Collectors.joining(System.lineSeparator()))); - PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, partitionsMap); + PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, newPartitions); try { applicationEventPublisher.publishEvent(event); } catch (Exception e) { @@ -480,6 +486,15 @@ public class HashPartitionService implements PartitionService { } } + private Set toTpiList(QueueKey queueKey, List partitions) { + if (partitions == null) { + return Collections.emptySet(); + } + return partitions.stream() + .map(partition -> buildTopicPartitionInfo(queueKey, partition)) + .collect(Collectors.toSet()); + } + @Override public Set getAllServiceIds(ServiceType serviceType) { return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet()); @@ -508,7 +523,6 @@ public class HashPartitionService implements PartitionService { return result; } - @Override public int resolvePartitionIndex(UUID entityId, int partitions) { int hash = hash(entityId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b0e1229f97..5bbff7663c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -45,6 +45,8 @@ public interface PartitionService { List getMyPartitions(QueueKey queueKey); + String getTopic(QueueKey queueKey); + /** * Received from the Discovery service when network topology is changed. * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java index b991e4614c..3f8926ad18 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java @@ -23,6 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; +import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.CF_STATES_QUEUE_NAME; + @Data @AllArgsConstructor public class QueueKey { @@ -32,6 +35,9 @@ public class QueueKey { private final String queueName; private final TenantId tenantId; + public static final QueueKey CF = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + public static final QueueKey CF_STATES = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_STATES_QUEUE_NAME); + public QueueKey(ServiceType type, Queue queue) { this.type = type; this.queueName = queue.getName(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 57a4941981..a16d33c884 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -28,8 +28,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; - @ToString(callSuper = true) public class PartitionChangeEvent extends TbApplicationEvent { @@ -55,8 +53,8 @@ public class PartitionChangeEvent extends TbApplicationEvent { return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_CORE, DataConstants.EDGE_QUEUE_NAME); } - public Set getCalculatedFieldsPartitions() { - return partitionsMap.getOrDefault(CALCULATED_FIELD_QUEUE_KEY, Collections.emptySet()); + public Set getCfPartitions() { + return partitionsMap.getOrDefault(QueueKey.CF, Collections.emptySet()); } private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index a2edc35d94..fe2c8b8a90 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -23,12 +23,19 @@ import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.util.UUID; public class KafkaTbQueueMsg implements TbQueueMsg { + + private static final int UUID_LENGTH = 36; + private final UUID key; private final TbQueueMsgHeaders headers; private final byte[] data; public KafkaTbQueueMsg(ConsumerRecord record) { - this.key = UUID.fromString(record.key()); + if (record.key().length() == UUID_LENGTH) { + this.key = UUID.fromString(record.key()); + } else { + this.key = UUID.randomUUID(); + } TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); record.headers().forEach(header -> { headers.put(header.key(), header.value()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index ef79834735..1d39a3bd0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -18,9 +18,11 @@ package org.thingsboard.server.queue.kafka; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; import org.springframework.util.StopWatch; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueMsg; @@ -29,9 +31,12 @@ import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. @@ -46,10 +51,15 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private final TbKafkaConsumerStatsService statsService; private final String groupId; + private final boolean readFromBeginning; // reset offset to beginning + private final boolean stopWhenRead; // stop consuming when reached initial end offsets + private Map endOffsets; // needed if stopWhenRead is true + @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, - TbQueueAdmin admin, TbKafkaConsumerStatsService statsService) { + TbQueueAdmin admin, TbKafkaConsumerStatsService statsService, + boolean readFromBeginning, boolean stopWhenRead) { super(topic); Properties props = settings.toConsumerProps(topic); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); @@ -67,13 +77,35 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; + this.readFromBeginning = readFromBeginning; + this.stopWhenRead = stopWhenRead; } @Override protected void doSubscribe(List topicNames) { if (!topicNames.isEmpty()) { topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); + if (readFromBeginning || stopWhenRead) { + consumer.subscribe(topicNames, new ConsumerRebalanceListener() { + @Override + public void onPartitionsRevoked(Collection partitions) {} + + @Override + public void onPartitionsAssigned(Collection partitions) { + log.debug("Handling onPartitionsAssigned {}", partitions); + if (readFromBeginning) { + consumer.seekToBeginning(partitions); + } + if (stopWhenRead) { + endOffsets = consumer.endOffsets(partitions).entrySet().stream() + .filter(entry -> entry.getValue() > 0) + .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); + } + } + }); + } else { + consumer.subscribe(topicNames); + } } else { log.info("unsubscribe due to empty topic list"); consumer.unsubscribe(); @@ -92,13 +124,32 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue stopWatch.stop(); log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); + List> recordList; if (records.isEmpty()) { - return Collections.emptyList(); + recordList = Collections.emptyList(); } else { - List> recordList = new ArrayList<>(256); - records.forEach(recordList::add); - return recordList; + recordList = new ArrayList<>(256); + records.forEach(record -> { + recordList.add(record); + if (stopWhenRead && endOffsets != null) { + int partition = record.partition(); + Long endOffset = endOffsets.get(partition); + if (endOffset == null) { + log.warn("End offset not found for {} [{}]", record.topic(), partition); + return; + } + log.trace("[{}-{}] Got record offset {}, expected end offset: {}", record.topic(), partition, record.offset(), endOffset - 1); + if (record.offset() >= endOffset - 1) { + endOffsets.remove(partition); + } + } + }); + } + if (stopWhenRead && endOffsets != null && endOffsets.isEmpty()) { + log.info("Reached end offset for {}, stopping consumer", consumer.assignment()); + stop(); } + return recordList; } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index 3c9b85e925..de4fc7049d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -97,9 +97,12 @@ public class TbKafkaProducerTemplate implements TbQueuePro @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + send(tpi, msg.getKey().toString(), msg, callback); + } + + public void send(TopicPartitionInfo tpi, String key, T msg, TbQueueCallback callback) { try { createTopicIfNotExist(tpi); - String key = msg.getKey().toString(); byte[] data = msg.getData(); ProducerRecord record; List
        headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); @@ -116,7 +119,7 @@ public class TbKafkaProducerTemplate implements TbQueuePro if (callback != null) { callback.onFailure(exception); } else { - log.warn("Producer template failure: {}", exception.getMessage(), exception); + log.warn("Producer template failure", exception); } } }); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index cdd0add38b..7213cf34b8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -54,6 +54,8 @@ public class TbKafkaTopicConfigs { private String housekeeperReprocessingProperties; @Value("${queue.kafka.topic-properties.calculated-field:}") private String calculatedFieldProperties; + @Value("${queue.kafka.topic-properties.calculated-field-state:}") + private String calculatedFieldStateProperties; @Getter private Map coreConfigs; @@ -83,6 +85,8 @@ public class TbKafkaTopicConfigs { private Map edgeEventConfigs; @Getter private Map calculatedFieldConfigs; + @Getter + private Map calculatedFieldStateConfigs; @PostConstruct private void init() { @@ -102,6 +106,7 @@ public class TbKafkaTopicConfigs { edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); calculatedFieldConfigs = PropertyUtils.getProps(calculatedFieldProperties); + calculatedFieldStateConfigs = PropertyUtils.getProps(calculatedFieldStateProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index c26e2d15c9..8e36ae4962 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -159,12 +160,12 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { + public TbQueueConsumer> createCalculatedFieldStateConsumer() { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); } @Override - public TbQueueProducer> createCalculatedFieldStateProducer() { + public TbQueueProducer> createCalculatedFieldStateProducer() { return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 01e174023c..99a50b2f5a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -100,6 +100,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin cfStateAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -142,6 +143,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); } @Override @@ -546,26 +548,28 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - consumerBuilder.clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(cfAdmin); - consumerBuilder.statsService(consumerStatsService); - return consumerBuilder.build(); + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateMsgProto.parseFrom(msg.getData()), msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); } @Override - public TbQueueProducer> createCalculatedFieldStateProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - requestBuilder.admin(cfAdmin); - return requestBuilder.build(); + public TbQueueProducer> createCalculatedFieldStateProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) + .admin(cfStateAdmin) + .build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 46b35f9acd..4fbf5c3e45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,7 +23,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -87,6 +87,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin cfStateAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -119,6 +120,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); } @Override @@ -338,26 +340,28 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - consumerBuilder.clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(cfAdmin); - consumerBuilder.statsService(consumerStatsService); - return consumerBuilder.build(); + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateMsgProto.parseFrom(msg.getData()), msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); } @Override - public TbQueueProducer> createCalculatedFieldStateProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); - requestBuilder.admin(cfAdmin); - return requestBuilder.build(); + public TbQueueProducer> createCalculatedFieldStateProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) + .admin(cfStateAdmin) + .build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index d3c1d09399..5f0af72955 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -17,7 +17,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -126,8 +126,8 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer(); - TbQueueConsumer> createCalculatedFieldStateConsumer(); + TbQueueConsumer> createCalculatedFieldStateConsumer(); - TbQueueProducer> createCalculatedFieldStateProducer(); + TbQueueProducer> createCalculatedFieldStateProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index dea6d9ed9e..a24f0663b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

        * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

        + * http://www.apache.org/licenses/LICENSE-2.0 + *

        * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,7 +40,6 @@ public @interface AfterStartUp { int CF_READ_PROFILE_ENTITIES_SERVICE = 10; int CF_READ_CF_SERVICE = 11; - int CF_STATE_RESTORE_SERVICE = 12; int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; From eae72065ee2dab301efc274e40154ea67302d83d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Feb 2025 13:59:28 +0200 Subject: [PATCH 277/416] fixed integration tests and added debug events creation --- ...CalculatedFieldEntityMessageProcessor.java | 55 ++-- .../CalculatedFieldStateException.java | 2 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 22 +- .../cf/CalculatedFieldIntegrationTest.java | 284 ++++++++++-------- 4 files changed, 198 insertions(+), 165 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index bfed515eb5..66b32dfbfe 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -197,12 +197,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state = getOrInitState(ctx); justRestored = true; } - if (state.updateState(newArgValues) || justRestored) { - cfIdList = new ArrayList<>(cfIdList); - cfIdList.add(ctx.getCfId()); - processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); - } else { - callback.onSuccess(CALLBACKS_PER_CF); + try { + if (state.updateState(newArgValues) || justRestored) { + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); + } } } @@ -212,37 +218,38 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state != null) { return state; } else { - ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - state = stateFuture.get(1, TimeUnit.MINUTES); - states.put(ctx.getCfId(), state); + try { + ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + state = stateFuture.get(1, TimeUnit.MINUTES); + states.put(ctx.getCfId(), state); + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, null, null, null, null, e); + } + throw new RuntimeException(e); + } } return state; } @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); - if (state.isReady() && ctx.isInitialized()) { - try { + try { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); } - } catch (Exception e) { - if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); - } + } else { + callback.onSuccess(); // State was updated but no calculation performed; } - } else { - callback.onSuccess(); // State was updated but no calculation performed; - } - try { cfStateService.persistState(ctxId, state, callback); } catch (Exception e) { if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { diff --git a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java index 6248ac1536..50ab512cb9 100644 --- a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java +++ b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.exception; -public class CalculatedFieldStateException extends Exception { +public class CalculatedFieldStateException extends RuntimeException { public CalculatedFieldStateException(String message) { super(message); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 0834b5583e..cdad1145e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -25,6 +25,7 @@ import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.exception.CalculatedFieldStateException; import java.util.ArrayList; import java.util.List; @@ -85,7 +86,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public boolean updateEntry(ArgumentEntry entry) { + public boolean updateEntry(ArgumentEntry entry) throws CalculatedFieldStateException { if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { updateTsRollingEntry(tsRollingEntry); } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { @@ -107,15 +108,18 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } private void addTsRecord(Long ts, KvEntry value) { - switch (value.getDataType()) { - case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); - case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); - case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); - case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); - case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); - //TODO: try catch + try { + switch (value.getDataType()) { + case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); + case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); + case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); + case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + } + cleanupExpiredRecords(); + } catch (Exception e) { + throw new IllegalArgumentException("Time series rolling arguments supports only numeric values."); } - cleanupExpiredRecords(); } private void addTsRecord(Long ts, double value) { diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 91aad7b1cd..f544f463fa 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -40,8 +40,10 @@ import org.thingsboard.server.controller.CalculatedFieldControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; @DaoSqlTest public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { @@ -84,20 +86,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> perform initial calculation CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + }); // update telemetry -> recalculate state doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); // update CF output -> perform calculation with updated output Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); @@ -106,32 +110,35 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes savedOutput.setName("temperatureF"); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + }); // update CF argument -> perform calculation with new argument Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); // update CF expression -> perform calculation with new expression savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); } @Test @@ -164,20 +171,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> state is not ready -> no calculation performed CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); // update telemetry -> perform calculation doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); } @Test @@ -211,20 +220,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> perform initial calculation with default value CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + }); // update telemetry -> recalculate state doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); } @Test @@ -275,93 +286,100 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF and perform initial calculation doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - // result of asset 1 - ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); - // result of asset 2 - ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + }); // update device telemetry -> recalculate state for all assets doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); - Thread.sleep(300); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); - - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); // update asset 1 telemetry -> recalculate state only for asset 1 doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); - // result of asset 2 (no changes) - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + // result of asset 2 (no changes) + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); // update asset 2 telemetry -> recalculate state only for asset 2 doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); - Thread.sleep(300); - - // result of asset 1 (no changes) - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 (no changes) + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + }); // add new entity to profile -> calculate state for new entity Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); - Thread.sleep(300); - - // result of asset 3 - ArrayNode z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + Asset finalAsset3 = asset3; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + }); // update device telemetry -> recalculate state for all assets doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); - // result of asset 3 - z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); // update profile for asset 3 -> delete state for asset 3 AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); @@ -371,22 +389,24 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // update device telemetry -> recalculate state for asset 1 and asset 2 doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); - - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); - - // no changes for asset 3 - z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + Asset updatedAsset3 = asset3; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); + + // no changes for asset 3 + ArrayNode z3 = getServerAttributes(updatedAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); } @Test @@ -421,20 +441,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> ctx is not initialized -> no calculation perform CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); // update telemetry -> ctx is not initialized -> no calculation perform doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); } private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { From 8d68f2bdbb619933384f3d49ce6ea9252733dc34 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Thu, 13 Feb 2025 14:44:33 +0200 Subject: [PATCH 278/416] UI: Imporove description for scada symbols --- .../main/data/json/system/scada_symbols/control-panel-hp.svg | 2 +- .../data/json/system/scada_symbols/horizontal-connector-hp.svg | 2 +- .../src/main/data/json/system/scada_symbols/leak-sensor.svg | 2 +- .../json/system/scada_symbols/left-analog-water-level-meter.svg | 2 +- .../src/main/data/json/system/scada_symbols/left-drain-pipe.svg | 2 +- .../data/json/system/scada_symbols/left-elbow-drain-pipe.svg | 2 +- .../main/data/json/system/scada_symbols/long-bottom-filter.svg | 2 +- .../json/system/scada_symbols/long-horizontal-connector-hp.svg | 2 +- .../src/main/data/json/system/scada_symbols/long-top-filter.svg | 2 +- .../json/system/scada_symbols/long-vertical-connector-hp.svg | 2 +- .../system/scada_symbols/right-analog-water-level-meter.svg | 2 +- .../main/data/json/system/scada_symbols/right-drain-pipe.svg | 2 +- .../data/json/system/scada_symbols/right-elbow-drain-pipe.svg | 2 +- .../main/data/json/system/scada_symbols/short-bottom-filter.svg | 2 +- .../data/json/system/scada_symbols/short-left-drain-pipe.svg | 2 +- .../data/json/system/scada_symbols/short-right-drain-pipe.svg | 2 +- .../main/data/json/system/scada_symbols/short-top-filter.svg | 2 +- .../data/json/system/scada_symbols/vertical-connector-hp.svg | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/application/src/main/data/json/system/scada_symbols/control-panel-hp.svg b/application/src/main/data/json/system/scada_symbols/control-panel-hp.svg index 630d0faa28..d9b0857156 100644 --- a/application/src/main/data/json/system/scada_symbols/control-panel-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/control-panel-hp.svg @@ -1,6 +1,6 @@ { "title": "HP Control panel", - "description": "Control panel", + "description": "Sends the command to the device or updates attribute/time series when the user pushes the button. Widget settings will enable you to configure behavior, how to fetch the initial state, and what to trigger when power on/off states.", "searchTags": [ "control" ], diff --git a/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg index 3548d687c5..33b6c0a222 100644 --- a/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="200" fill="none" version="1.1" viewBox="0 0 200 200"><tb:metadata xmlns=""><![CDATA[{ "title": "HP Horizontal connector", - "description": "Horizontal connector", + "description": "Horizontal connector with an optional directional arrow to visually indicate flow.", "widgetSizeX": 1, "widgetSizeY": 1, "tags": [ diff --git a/application/src/main/data/json/system/scada_symbols/leak-sensor.svg b/application/src/main/data/json/system/scada_symbols/leak-sensor.svg index e2859ea90c..0fce8be608 100644 --- a/application/src/main/data/json/system/scada_symbols/leak-sensor.svg +++ b/application/src/main/data/json/system/scada_symbols/leak-sensor.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="100" height="100" fill="none" version="1.1" viewBox="0 0 100 100"><tb:metadata xmlns=""><![CDATA[{ "title": "Leak sensor", - "description": "Leak sensor", + "description": "Leak sensor for real-time detection and warning of fluid leakage.", "searchTags": [ "leak" ], diff --git a/application/src/main/data/json/system/scada_symbols/left-analog-water-level-meter.svg b/application/src/main/data/json/system/scada_symbols/left-analog-water-level-meter.svg index 18394d3d2c..2cdc9e587a 100644 --- a/application/src/main/data/json/system/scada_symbols/left-analog-water-level-meter.svg +++ b/application/src/main/data/json/system/scada_symbols/left-analog-water-level-meter.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="200" fill="none" version="1.1" viewBox="0 0 200 200"><tb:metadata xmlns=""><![CDATA[{ "title": "Left analog water level meter", - "description": "Left analog water level meter", + "description": "Left analog water level meter with real-time display of fluid levels.", "searchTags": [ "water meter" ], diff --git a/application/src/main/data/json/system/scada_symbols/left-drain-pipe.svg b/application/src/main/data/json/system/scada_symbols/left-drain-pipe.svg index 326a263208..cb7bb26821 100644 --- a/application/src/main/data/json/system/scada_symbols/left-drain-pipe.svg +++ b/application/src/main/data/json/system/scada_symbols/left-drain-pipe.svg @@ -1,7 +1,7 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="400" height="200" fill="none" version="1.1" viewBox="0 0 400 200"> <tb:metadata><![CDATA[{ "title": "Left drain pipe", - "description": "Left drain pipe", + "description": "Left drain pipe with configurable fluid and leak visualizations.", "searchTags": [ "pipe", "drain" diff --git a/application/src/main/data/json/system/scada_symbols/left-elbow-drain-pipe.svg b/application/src/main/data/json/system/scada_symbols/left-elbow-drain-pipe.svg index b9a6aa6057..085830c5e4 100644 --- a/application/src/main/data/json/system/scada_symbols/left-elbow-drain-pipe.svg +++ b/application/src/main/data/json/system/scada_symbols/left-elbow-drain-pipe.svg @@ -1,7 +1,7 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="400" height="200" fill="none" version="1.1" viewBox="0 0 400 200"> <tb:metadata xmlns=""><![CDATA[{ "title": "Left elbow drain pipe", - "description": "Left elbow drain pipe", + "description": "Left elbow drain pipe with configurable fluid and leak visualizations.", "searchTags": [ "pipe", "drain" diff --git a/application/src/main/data/json/system/scada_symbols/long-bottom-filter.svg b/application/src/main/data/json/system/scada_symbols/long-bottom-filter.svg index d9defd33d9..689851529d 100644 --- a/application/src/main/data/json/system/scada_symbols/long-bottom-filter.svg +++ b/application/src/main/data/json/system/scada_symbols/long-bottom-filter.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="600" fill="none" version="1.1" viewBox="0 0 200 600"> <tb:metadata xmlns=""><![CDATA[{ - "title": "Long bottom filter", + "title": "Long bottom filter with configurable click actions for custom operations, dashboard manipulation, etc.", "description": "Long bottom filter", "searchTags": [ "filter" diff --git a/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg index 4376bb2348..86bb07a520 100644 --- a/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg @@ -1,7 +1,7 @@ <svg width="400" height="200" fill="none" version="1.1" viewBox="0 0 400 200" xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg"> <tb:metadata><![CDATA[{ "title": "HP Long horizontal connector", - "description": "Long horizontal connector", + "description": "Long horizontal connector with an optional directional arrow to visually indicate flow.", "widgetSizeX": 2, "widgetSizeY": 1, "tags": [ diff --git a/application/src/main/data/json/system/scada_symbols/long-top-filter.svg b/application/src/main/data/json/system/scada_symbols/long-top-filter.svg index a63b679f0a..9b287cbe46 100644 --- a/application/src/main/data/json/system/scada_symbols/long-top-filter.svg +++ b/application/src/main/data/json/system/scada_symbols/long-top-filter.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="600" fill="none" version="1.1" viewBox="0 0 200 600"> <tb:metadata xmlns=""><![CDATA[{ - "title": "Long top filter", + "title": "Long top filter with configurable click actions for custom operations, dashboard manipulation, etc.", "description": "Title\nLong top filter\n", "searchTags": [ "filter" diff --git a/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg index 86235ff1c1..d7a0aa8aa7 100644 --- a/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="400" fill="none" version="1.1" viewBox="0 0 200 400"><tb:metadata xmlns=""><![CDATA[{ "title": "HP Long vertical connector", - "description": "Long vertical connector", + "description": "Long vertical connector with an optional directional arrow to visually indicate flow.", "widgetSizeX": 1, "widgetSizeY": 2, "tags": [ diff --git a/application/src/main/data/json/system/scada_symbols/right-analog-water-level-meter.svg b/application/src/main/data/json/system/scada_symbols/right-analog-water-level-meter.svg index 4bd6754f5c..1b1be4f007 100644 --- a/application/src/main/data/json/system/scada_symbols/right-analog-water-level-meter.svg +++ b/application/src/main/data/json/system/scada_symbols/right-analog-water-level-meter.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="200" fill="none" version="1.1" viewBox="0 0 200 200"><tb:metadata xmlns=""><![CDATA[{ "title": "Right analog water level meter", - "description": "Right analog water level meter", + "description": "Right analog water level meter with real-time display of fluid levels.", "searchTags": [ "water meter" ], diff --git a/application/src/main/data/json/system/scada_symbols/right-drain-pipe.svg b/application/src/main/data/json/system/scada_symbols/right-drain-pipe.svg index 093078c69b..a990e023e1 100644 --- a/application/src/main/data/json/system/scada_symbols/right-drain-pipe.svg +++ b/application/src/main/data/json/system/scada_symbols/right-drain-pipe.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="400" height="200" fill="none" version="1.1" viewBox="0 0 400 200"><tb:metadata xmlns=""><![CDATA[{ "title": "Right drain pipe", - "description": "Right drain pipe", + "description": "Right drain pipe with configurable fluid and leak visualizations.", "searchTags": [ "pipe", "drain" diff --git a/application/src/main/data/json/system/scada_symbols/right-elbow-drain-pipe.svg b/application/src/main/data/json/system/scada_symbols/right-elbow-drain-pipe.svg index 63b143eeef..6f5ae5abb9 100644 --- a/application/src/main/data/json/system/scada_symbols/right-elbow-drain-pipe.svg +++ b/application/src/main/data/json/system/scada_symbols/right-elbow-drain-pipe.svg @@ -1,7 +1,7 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="400" height="200" fill="none" version="1.1" viewBox="0 0 400 200"> <tb:metadata xmlns=""><![CDATA[{ "title": "Right elbow drain pipe", - "description": "Right elbow drain pipe", + "description": "Right elbow drain pipe with configurable fluid and leak visualizations.", "searchTags": [ "pipe", "drain" diff --git a/application/src/main/data/json/system/scada_symbols/short-bottom-filter.svg b/application/src/main/data/json/system/scada_symbols/short-bottom-filter.svg index b3235c5906..cf191d40fe 100644 --- a/application/src/main/data/json/system/scada_symbols/short-bottom-filter.svg +++ b/application/src/main/data/json/system/scada_symbols/short-bottom-filter.svg @@ -1,7 +1,7 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="400" fill="none" version="1.1" viewBox="0 0 200 400"> <tb:metadata xmlns=""><![CDATA[{ "title": "Short bottom filter", - "description": "Short bottom filter", + "description": "Short bottom filter with configurable click actions for custom operations, dashboard manipulation, etc.", "searchTags": [ "filter" ], diff --git a/application/src/main/data/json/system/scada_symbols/short-left-drain-pipe.svg b/application/src/main/data/json/system/scada_symbols/short-left-drain-pipe.svg index abf43aacb9..da2273d096 100644 --- a/application/src/main/data/json/system/scada_symbols/short-left-drain-pipe.svg +++ b/application/src/main/data/json/system/scada_symbols/short-left-drain-pipe.svg @@ -1,7 +1,7 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="200" fill="none" version="1.1" viewBox="0 0 200 200"> <tb:metadata><![CDATA[{ "title": "Short left drain pipe", - "description": "Short left drain pipe", + "description": "Short left drain pipe with configurable fluid and leak visualizations.", "searchTags": [ "pipe", "drain" diff --git a/application/src/main/data/json/system/scada_symbols/short-right-drain-pipe.svg b/application/src/main/data/json/system/scada_symbols/short-right-drain-pipe.svg index 1b2ae3934a..1e01fe8b77 100644 --- a/application/src/main/data/json/system/scada_symbols/short-right-drain-pipe.svg +++ b/application/src/main/data/json/system/scada_symbols/short-right-drain-pipe.svg @@ -1,7 +1,7 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="200" fill="none" version="1.1" viewBox="0 0 200 200"> <tb:metadata><![CDATA[{ "title": "Short right drain pipe", - "description": "Short right drain pipe", + "description": "Short right drain pipe with configurable fluid and leak visualizations.", "searchTags": [ "pipe", "drain" diff --git a/application/src/main/data/json/system/scada_symbols/short-top-filter.svg b/application/src/main/data/json/system/scada_symbols/short-top-filter.svg index d50eada2aa..0a77c78f00 100644 --- a/application/src/main/data/json/system/scada_symbols/short-top-filter.svg +++ b/application/src/main/data/json/system/scada_symbols/short-top-filter.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="400" fill="none" version="1.1" viewBox="0 0 200 400"> <tb:metadata xmlns=""><![CDATA[{ - "title": "Short top filter", + "title": "Short top filter with configurable click actions for custom operations, dashboard manipulation, etc.", "description": "Short top filter", "searchTags": [ "filter" diff --git a/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg index aea94b0dd2..6ce9336fee 100644 --- a/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="200" height="200" fill="none" version="1.1" viewBox="0 0 200 200"><tb:metadata xmlns=""><![CDATA[{ "title": "HP Vertical connector", - "description": "Vertical connector", + "description": "Vertical connector with an optional directional arrow to visually indicate flow.", "widgetSizeX": 1, "widgetSizeY": 1, "tags": [ From d2389817a48d69063f18fcc705b65131e09162ad Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko <Dgerik2015@gmail.com> Date: Thu, 13 Feb 2025 15:00:44 +0200 Subject: [PATCH 279/416] UI: Fixed scada bundle order and add hook to general --- .../widget_bundles/general_high_performance_scada_symbols.json | 1 + .../widget_bundles/high_performance_scada_fluid_system.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/data/json/system/widget_bundles/general_high_performance_scada_symbols.json b/application/src/main/data/json/system/widget_bundles/general_high_performance_scada_symbols.json index 33efaacc0c..97768915f7 100644 --- a/application/src/main/data/json/system/widget_bundles/general_high_performance_scada_symbols.json +++ b/application/src/main/data/json/system/widget_bundles/general_high_performance_scada_symbols.json @@ -29,6 +29,7 @@ "hp_top_tee_connector", "hp_drawwork", "hp_crane", + "hp_hook", "hp_consumers", "hp_house", "hp_apartments", diff --git a/application/src/main/data/json/system/widget_bundles/high_performance_scada_fluid_system.json b/application/src/main/data/json/system/widget_bundles/high_performance_scada_fluid_system.json index 1e41b22446..29a0571dfa 100644 --- a/application/src/main/data/json/system/widget_bundles/high_performance_scada_fluid_system.json +++ b/application/src/main/data/json/system/widget_bundles/high_performance_scada_fluid_system.json @@ -5,7 +5,7 @@ "image": "tb-image:aHBfc2NhZGFfZmx1aWRfc3lzdGVtX2J1bmRsZV9pbWFnZS5wbmc=:IkhpZ2gtcGVyZm9ybWFuY2UgU0NBREEgZmx1aWQgc3lzdGVtIiBzeXN0ZW0gYnVuZGxlIGltYWdl:SU1BR0U=;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAACgCAMAAAB+IdObAAABNVBMVEXe3t7f39/X19cAAADf39/e3t7f39/e3t7e3t7////I3/fr6+vHx8esrKzj4+OcrsHT0tKVlZWvr6/Kysrx8fF0dHT9/f24uLjBwcHc29v39/etra27u7ubm5vt7e2/v7/Dw8OysrKqqqq+1ev6+vrE2/Lh4eHV1dWoqKiQkJCmpqa1yt+2trbu7e2YmJi/1eyurq7B2O9ycXFlZWXT09Ojo6PMzMzIyMjm5eWBgYGenp78/Py0tLSTk5OLi4uwxdq6z+Wous6ioqKGhoZjY2OgoKBoaGheXl5WVlasv9SorrR/fn7R0dGlpaVSUlKhrr13d3evuseTpLWWoq+fn59sbGw+Pj6rv9Tv8veitci4uryAk7vf5O7P1+WQocRgeasvLy8hISG/yd2frsx/k7tvhrNAXZmZOz+XAAAACHRSTlPvICAAv7Cvv9cI9yEAAAzySURBVHja7JttT9pQGIaNe8lTejg9dDRqJEElwFxNN1lIZNko3RrRuoAvUaOf/P+/YkecPq0v0PY5lZZ48dUvV06vm54Ylt4tv1+CovNx+d3ScvE1JFLjAywE8R6rNcg9SzNEama36/R/raxLVmazsfrTrMFcmCYiNrdAb7VX6ht6u91e0+MhdkwGBNSLdDd0ZK/V0mOzYwIBxSLiJ+hhEc71+EBdAAGVIuKzTuIzwUSpyKoeZa/FE5oAAXUijo609D0UiQ+hE3UiTOhIvf5Jb0kSimwIoEIX2dTDIp9wfpOwCXGpHHyfwqiSXgSHNzy/mR1J5RtMYT29SE1HEseOmHMX+aIroT53kdVnToTvJRbZmbtINJE1+bl9tAooYpTCGIZbakhKSTFyJnLlHZZs2x4UXmRC8+vXRRHZXQwRyWKILMCJNO2GvRAiJds7VDy/XZqImVLEHaie35s/LCrSX59CPyrChhe5mV+LW1ZYhVUi+H4lAgNEHPWqVm5itzSN+87LybwcRd9nmmblZn4tTbJ1YCYVMc9XNEmuTuSW7WOWRIQNt6saRWTQcKXIblO5iMatIzZbBOPQNJLIlYHzq1REwg+ceCLObRxEEdt2cX7VikxSqc0WMc+3NMTKz/xqYbb32XQRti/j+E8uY9cwlSki4qjDtRC5m98wbUzlsYjTmcRRiBO5S4U9J2IOMQ6yiDvwshfRqg+poAg7xjhUiGQ4v09TQRFx5GMcKkQuB3amIgg7cFCkG42jAPMbSWXI7kTM4Q/tnkLFfg+XqVQq4tE3R2HmNwzvXYwmcRT7RKSI3+v1MhFxB/YriOAKi8lLCVcu0jyV8zuwbfs1RH4MzfvLR1n5iTQbrza/nS7+kTPKZn53s3u0MA4BYXoWL2DsVbz4Rt7fiza/ZRnHM7BRuVAnwjoOPAJTaSubX9vNVoR3MI6nCEyFKpLp/OL7O/I0FTUijUGG81s+N2EmtVE5540w34FYOCOeY5FqD+OYBZOp5HR+q9tDBnHAi28uT6SGccTFHJVJV91LT70I8/uQAsfn6UUOPeXzi3EkhVkWTytiN1XP7wXGkRz25+a1G/HgBUwg0Y0tMi49It3/R04gW2aLnJSUcAY06CLf1Dxa10CDLiK8yPy6V6lEPCBCF4Egslqul2p+AyBCF4keSVN+Usyvy4AKXQQCeiMVoEMXgbOnIjne3iki4ow4vX+BAF0EEScYRTPpF2JjTPWgiyDB+OHSnvDR8n4DGboIwoKxO9nfRPPbOAsIx6FWBLkOxieeYRinEmM243EQXMNcWIgfuL6J5JE3kX/tneeu0zAUgEEsH8mx3MY0o0naRIiEqK0Q6oDS3h8gJCRA4gcgJJaY7/8I2Bk9GaVJKKOMD6Gbyy3H/myfeKSFU+O/yKnxj4nw/uzBIjY0yY07Dx9eXWuHwHOn1cigPwt/3lWEznyNQRFhrNchfBONZMyn8BOZrjqJPPEp7CNcU9Ek4sNPZdRehJ+5Ar4J1dhhEQo/Fae1iB8LOAilv1OEthSh9xk0wW6wkxd5EkEbAu3ERbCpG2BrccIifCugNWvWKEL38QtE+Ay6ELAGkRXsQYQ/UQQ9jjCpi9D9FfnpIlvoSiBOUWTfJCiCReDOegs36E8Y1Fn/aBERjkeSMRXfLfKE1YJGwU1Odty8Y9Zf4vxIEToynbxAZ9MPv0uEbqp1dM84qdA7q6pQ2kbk7ds3r941icwnlfKk13eIONWXu0nYobudbYLI37oPku/9auy1aBZ5+eHdS3h1WOR6psEN3/dHw8wp9rqKzBiUmDkq6MRl4N2GBBb0VAh6j1WyqFkE3nyCj68PivRp0prBZumAIlxqwySW3lEkKtfuPpcaQVrkLpTJHBXELZsYorlHvrx6/flQjogg0RipUHwKKSvfT2rGuoj4rBT3hgqbu9lzSOEAgVs3oUcnO1NRaV8ANlz6NaZqeLEOIuVM36raziHHyr6ogKxuEoruInUPf1eeZUOCqX4UYys3idQzJOayro91LMgqtJTo8erUuT5SZKIG0AQkaCDh6RSgfijaiiygQECTNvfQhKfxsb94/ANF/KSqnlkRsTzYmTgtRSgtzR+E9JjKDRUQ+2JqFTLoJgNE0GNEGE3TAE0sq9huIiaE83YipfaN08kxM8HGMSBjeuMmIfehgEYVd2oiId2xWtEcBiVcQugU6g3HIWOywho2iCwAYT4hdwBKJrYuf1tYTF92CS3VpQSFDpgqm3fFoYju5e3G1bGJ2UaEG+XAONI8fRcZPTwQi3K+U7pfhF2XvyTXQwbXc+ZiUDnc8fcUhwPATAZJ3EakzwCRCXADv7PSaX2FGTJnAKpLFgKQJ3tFBtg188G4cA1ISIutfZtnItgh6mKCWXJQxAWE3ZTfr1Y611M7007Hq14qit4kfALIep/IoCBy/S6KjAdooppkONRNa5pVe5rlZC5nKiHhELJqIfKsfE9/wLIq65ZKjzS+hR5ZVtwDZNbUI3URHFkaqAzUzUTCSEXyDsk2DiuVx80ii/Kk7gKOWWt6ezq7f3b/eRpQqqXcI+SsqL+3R+4mF04ick2KCMaYkCJ3YYD7GRxZti7DexaEdNTPytFZ5k4IZ00imEjY1IhlPTWXkhtMleQVat4DRPu2yNX3N3IR9uL9C6ZEsEsoITzEOJ6hjx2RnwbYppf7qkxqFgkAmWVeaGYuN+tgubwDlr4rzzyjZMhgh0P3iVyTF/Tq1athJgLsIVPNe21QHFlDKOKNpICdDDEsz7YiQkbNIhogvcwLRZb9W7duPVk+mHhZUH1uAXMIDWv333qP5CiRDCUyqN98cQVq3LolNYYGyy24afuEOEeKnC23UuTRcnJPl8xlSmY3N0qbRK7tEykPLT1taYTC9JaEwcaTWCtT1201lI8SsT2+4jeWaxl4uwywA9qK3G0WCasiI/CUiAWxxbk1tUHSWsQAZJFuFr2VPuf6bds2by2Xs0f3zWW2uMKaDxuH1mBOM5zJtcfjnMe1oZXdf2V36w8A4lu3eri07CISlyd2Vw6hbHNzW7fZ2VKhjaGIIacb0XTXGly7Zmdcu3b32u5yUBLpA9N1k6czoe456eqXQolRq2SfAfIA76vZ7ZY9DZabO2fGFMod5wISVETQJOPu3bvXdqCHmiDo0MrLM+QVc0DiLKGIuCnDNosMAYlwFTU1IcHUYip0sOa8tCZ5AIj/DZHB3UGKvM5c7qo/Km1GJvnqx0yL1YaU+oybhZabq+mmWSQqD35OS6sR3bO8dKniWTqfZrmuykee1UWQpNpKJQcK4H1Vz8ozwEzXvNzUp7gic9osUQQgZ8mmCT2mLN1XWdM8J7m+imQ3sqa1VkVnv8hKVlFkq+p8tWvmu0RbupiWZytdret+xE/3sVMd8gYCj+3WwbiLXAMSOs0iWbdABc4J2aBHUu4K18GS2xbv4c7qoAjxK+v4e+iRDSsF6m4ooQEgwRE7xE1y17QK7QYcKg3nYhUPi7hQ4J468tMho3AOYJvl3TDy7Mit7mbXbrqHxfV3L+Etd4ikTwFhMrO2Aj0kenknuiblDmGLI0TUMHUmUOgQmNupiYHt5rM2IpWF24bjIYkBeWQ0iWllqT8mx4hwjidwlpcYlBtObXRNaCfiisqJKe9hYDxBkaGVR/XM9P5RIuAQvAUXv2RHXSFGbBbhQfUslm9FIaKBk/2aVk4rgNLjRESc/51pLmLnJoa9wmPsBpF6uuNJNa+cKwNTzxucMexN9e4imAQkYjiEbX0nqWao1RRai/CgbnLzbOSBAg+wXRV3MYEiGj1GBE10E0wsLuXxSHlc7/TESpRD3+BE/bNEIaRwO3uoiM8bcL18tAiwmEjcZ3hSgA8V/Wm3Z4g+lIkfEInj3otoyBjdLtybSSW3AkpE9FgRhUh3/X6fpuFNMR/FPBkqrOvD0GrxTMVWcOfm7jKovWpGjhFBTJ8k3BxNJuNJ5PDsIXX35+xxTT281+ME4cMtBQQf0h0tgioIPlTsKoIbDEQY99wHQ2M8HvbkGNsT1SXHiyCm5nCSwZ1gggV2EuEx7IWFNGSwl4g2iHSGjkYOpY4zDo94Uw2NoAN1j9N54xmNunqcqAihbjePkxUhfMugHWJGyQmLEOI2VQOn/tMWIdQV0EjkEHLqIqQXOE2h1i4/EGADP5XHLUVcIceNBg3vjd8eEIkF/ESE3yyC7whkgcH2h9Eo2y30u+fZ8cw3pJUIjbDhjWp9Qk2jIr/uH/wETQPumO7FeNJvgLf93/dKFTccQ9ZdoRmBpoVQYEPJd4J7pTrXV6SBBpF4lBBpddY3JFqdaJRwg3QEPZpNuoss4HsZk248mUwieoBgMknT+dRFaNvP7PwFIph93UUi7Xs5I92IaCMRac25v+UTlecukr+CS+fO/xVdIjUunL9M/nTOXTl/4St0s1h4GZ5DIgAAAABJRU5ErkJggg==", "scada": true, "description": "Bundle with high-performance SCADA symbols for fluid system", - "order": 9350, + "order": 9415, "name": "High-performance SCADA fluid system" }, "widgetTypeFqns": [ From a460570c449adf9a11e23ed9ea4701a3872f0d06 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko <Dgerik2015@gmail.com> Date: Thu, 13 Feb 2025 15:24:12 +0200 Subject: [PATCH 280/416] UI: Fixed scale symbol units --- .../json/system/scada_symbols/dynamic-vertical-scale-hp.svg | 2 +- .../data/json/system/scada_symbols/simple-vertical-scale-hp.svg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg b/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg index 98185cbc8d..960abed340 100644 --- a/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg @@ -776,7 +776,7 @@ } ] } -Outdoor°C +Outdoor°C diff --git a/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg b/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg index bfc356cca2..fa0c0fc4d1 100644 --- a/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg @@ -704,7 +704,7 @@ } ] }]]> -Outdoor°C +Outdoor°C From f342bd989ef891be8a56e8ea35ba2ca4c0e024e1 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 13 Feb 2025 15:31:09 +0200 Subject: [PATCH 281/416] Minor improvements --- .../auth/rest/RestAuthenticationProvider.java | 1 - .../queue/discovery/HashPartitionService.java | 30 ++++++------------- .../server/queue/kafka/KafkaTbQueueMsg.java | 2 +- .../server/queue/util/AfterStartUp.java | 8 ++--- 4 files changed, 14 insertions(+), 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index 56215b82ac..b9fe54deec 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -106,7 +106,6 @@ public class RestAuthenticationProvider implements AuthenticationProvider { if (twoFactorAuthService.isTwoFaEnabled(securityUser.getTenantId(), securityUser.getId())) { return new MfaAuthenticationToken(securityUser); } else { - systemSecurityService.logLoginAction(securityUser, authentication.getDetails(), ActionType.LOGIN, null); } } else { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 249dfa859f..751e235adc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -228,7 +228,7 @@ public class HashPartitionService implements PartitionService { }); if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { publishPartitionChangeEvent(ServiceType.TB_RULE_ENGINE, queueKeys.stream() - .collect(Collectors.toMap(k -> k, k -> Collections.emptySet())), Collections.emptyMap()); + .collect(Collectors.toMap(k -> k, k -> Collections.emptySet()))); } } @@ -408,7 +408,6 @@ public class HashPartitionService implements PartitionService { myPartitions = newPartitions; Map> changedPartitionsMap = new HashMap<>(); - Map> oldPartitionsMap = new HashMap<>(); Set removed = new HashSet<>(); oldPartitions.forEach((queueKey, partitions) -> { @@ -429,16 +428,16 @@ public class HashPartitionService implements PartitionService { myPartitions.forEach((queueKey, partitions) -> { if (!partitions.equals(oldPartitions.get(queueKey))) { - changedPartitionsMap.put(queueKey, toTpiList(queueKey, partitions)); - oldPartitionsMap.put(queueKey, toTpiList(queueKey, oldPartitions.get(queueKey))); + Set tpiList = partitions.stream() + .map(partition -> buildTopicPartitionInfo(queueKey, partition)) + .collect(Collectors.toSet()); + changedPartitionsMap.put(queueKey, tpiList); } }); if (!changedPartitionsMap.isEmpty()) { changedPartitionsMap.entrySet().stream() .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) - .forEach((serviceType, partitionsMap) -> { - publishPartitionChangeEvent(serviceType, partitionsMap, oldPartitionsMap); - }); + .forEach(this::publishPartitionChangeEvent); } if (currentOtherServices == null) { @@ -470,15 +469,13 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } - private void publishPartitionChangeEvent(ServiceType serviceType, - Map> newPartitions, - Map> oldPartitions) { - log.info("Partitions changed: {}", System.lineSeparator() + newPartitions.entrySet().stream() + private void publishPartitionChangeEvent(ServiceType serviceType, Map> partitionsMap) { + log.info("Partitions changed: {}", System.lineSeparator() + partitionsMap.entrySet().stream() .map(entry -> "[" + entry.getKey() + "] - [" + entry.getValue().stream() .map(tpi -> tpi.getPartition().orElse(-1).toString()).sorted() .collect(Collectors.joining(", ")) + "]") .collect(Collectors.joining(System.lineSeparator()))); - PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, newPartitions); + PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, partitionsMap); try { applicationEventPublisher.publishEvent(event); } catch (Exception e) { @@ -486,15 +483,6 @@ public class HashPartitionService implements PartitionService { } } - private Set toTpiList(QueueKey queueKey, List partitions) { - if (partitions == null) { - return Collections.emptySet(); - } - return partitions.stream() - .map(partition -> buildTopicPartitionInfo(queueKey, partition)) - .collect(Collectors.toSet()); - } - @Override public Set getAllServiceIds(ServiceType serviceType) { return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index fe2c8b8a90..302f36ff1a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -31,7 +31,7 @@ public class KafkaTbQueueMsg implements TbQueueMsg { private final byte[] data; public KafkaTbQueueMsg(ConsumerRecord record) { - if (record.key().length() == UUID_LENGTH) { + if (record.key().length() <= UUID_LENGTH) { this.key = UUID.fromString(record.key()); } else { this.key = UUID.randomUUID(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index a24f0663b5..6943fa6753 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

        + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

        - * http://www.apache.org/licenses/LICENSE-2.0 - *

        + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From fc513c5a5c8b59e11af95f5822f92693639087ab Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 13 Feb 2025 15:39:38 +0200 Subject: [PATCH 282/416] Add bundles to widget-type-info --- .../server/controller/BaseController.java | 5 ++ .../controller/WidgetTypeController.java | 2 +- .../server/dao/widget/WidgetTypeService.java | 2 + .../common/data/widget/WidgetTypeInfo.java | 24 +++++++ .../server/dao/model/ModelConstants.java | 1 + .../dao/model/sql/WidgetTypeInfoEntity.java | 9 +++ .../dao/sql/widget/JpaWidgetTypeDao.java | 5 ++ .../mapping/AbstractEntityInfosConverter.java | 71 +++++++++++++++++++ .../WidgetBundleEntityInfosConverter.java | 29 ++++++++ .../server/dao/widget/WidgetTypeDao.java | 2 + .../dao/widget/WidgetTypeServiceImpl.java | 7 ++ .../sql/schema-views-and-functions.sql | 11 ++- 12 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/util/mapping/AbstractEntityInfosConverter.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/util/mapping/WidgetBundleEntityInfosConverter.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index dbe430ba18..2b248e57b3 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -125,6 +125,7 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.util.ThrowingBiFunction; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.asset.AssetProfileService; @@ -759,6 +760,10 @@ public abstract class BaseController { return checkEntityId(widgetTypeId, widgetTypeService::findWidgetTypeDetailsById, operation); } + WidgetTypeInfo checkWidgetTypeInfoId(WidgetTypeId widgetTypeId, Operation operation) throws ThingsboardException { + return checkEntityId(widgetTypeId, widgetTypeService::findWidgetTypeInfoById, operation); + } + Dashboard checkDashboardId(DashboardId dashboardId, Operation operation) throws ThingsboardException { return checkEntityId(dashboardId, dashboardService::findDashboardById, operation); } diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java index e4973a71c3..2c61ecac79 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java @@ -118,7 +118,7 @@ public class WidgetTypeController extends AutoCommitController { @PathVariable("widgetTypeId") String strWidgetTypeId) throws ThingsboardException { checkParameter("widgetTypeId", strWidgetTypeId); WidgetTypeId widgetTypeId = new WidgetTypeId(toUUID(strWidgetTypeId)); - return new WidgetTypeInfo(checkWidgetTypeId(widgetTypeId, Operation.READ)); + return checkWidgetTypeInfoId(widgetTypeId, Operation.READ); } @ApiOperation(value = "Create Or Update Widget Type (saveWidgetType)", diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeService.java index 9cad249787..c592777142 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeService.java @@ -35,6 +35,8 @@ public interface WidgetTypeService extends EntityDaoService { WidgetTypeDetails findWidgetTypeDetailsById(TenantId tenantId, WidgetTypeId widgetTypeId); + WidgetTypeInfo findWidgetTypeInfoById(TenantId tenantId, WidgetTypeId widgetTypeId); + boolean widgetTypeExistsByTenantIdAndWidgetTypeId(TenantId tenantId, WidgetTypeId widgetTypeId); WidgetTypeDetails saveWidgetType(WidgetTypeDetails widgetType); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeInfo.java index 1ce8ad6ddc..2ea98f0c17 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeInfo.java @@ -16,13 +16,22 @@ package org.thingsboard.server.common.data.widget; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; import lombok.Data; +import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.validation.NoXss; +import java.io.Serial; +import java.util.Collections; +import java.util.List; + @Data public class WidgetTypeInfo extends BaseWidgetType { + @Serial + private static final long serialVersionUID = 1343617007959780969L; + @Schema(description = "Base64 encoded widget thumbnail", accessMode = Schema.AccessMode.READ_ONLY) private String image; @NoXss @@ -34,6 +43,9 @@ public class WidgetTypeInfo extends BaseWidgetType { @NoXss @Schema(description = "Type of the widget (timeseries, latest, control, alarm or static)", accessMode = Schema.AccessMode.READ_ONLY) private String widgetType; + @Valid + @Schema(description = "Bundles", accessMode = Schema.AccessMode.READ_ONLY) + private List bundles; public WidgetTypeInfo() { super(); @@ -53,6 +65,16 @@ public class WidgetTypeInfo extends BaseWidgetType { this.description = widgetTypeInfo.getDescription(); this.tags = widgetTypeInfo.getTags(); this.widgetType = widgetTypeInfo.getWidgetType(); + this.bundles = Collections.emptyList(); + } + + public WidgetTypeInfo(WidgetTypeInfo widgetTypeInfo, List bundles) { + super(widgetTypeInfo); + this.image = widgetTypeInfo.getImage(); + this.description = widgetTypeInfo.getDescription(); + this.tags = widgetTypeInfo.getTags(); + this.widgetType = widgetTypeInfo.getWidgetType(); + this.bundles = bundles; } public WidgetTypeInfo(WidgetTypeDetails widgetTypeDetails) { @@ -65,5 +87,7 @@ public class WidgetTypeInfo extends BaseWidgetType { } else { this.widgetType = ""; } + this.bundles = Collections.emptyList(); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index d404c3128c..9628935b55 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -312,6 +312,7 @@ public class ModelConstants { public static final String WIDGETS_BUNDLE_SCADA_PROPERTY = "scada"; public static final String WIDGETS_BUNDLE_DESCRIPTION = "description"; public static final String WIDGETS_BUNDLE_ORDER = "widgets_bundle_order"; + public static final String WIDGET_BUNDLES_PROPERTY = "bundles"; /** * Widget_type constants. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeInfoEntity.java index 7a9e7f35bc..c503632b91 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeInfoEntity.java @@ -17,17 +17,21 @@ package org.thingsboard.server.dao.model.sql; import io.hypersistence.utils.hibernate.type.array.StringArrayType; import jakarta.persistence.Column; +import jakarta.persistence.Convert; import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.hibernate.annotations.Immutable; import org.hibernate.annotations.Type; +import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.widget.BaseWidgetType; import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.util.mapping.WidgetBundleEntityInfosConverter; import java.util.HashMap; +import java.util.List; import java.util.Map; @Data @@ -58,6 +62,10 @@ public class WidgetTypeInfoEntity extends AbstractWidgetTypeEntity bundles; + public WidgetTypeInfoEntity() { super(); } @@ -70,6 +78,7 @@ public class WidgetTypeInfoEntity extends AbstractWidgetTypeEntity findSystemWidgetTypes(WidgetTypeFilter widgetTypeFilter, PageLink pageLink) { boolean deprecatedFilterEnabled = !DeprecatedFilter.ALL.equals(widgetTypeFilter.getDeprecatedFilter()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/mapping/AbstractEntityInfosConverter.java b/dao/src/main/java/org/thingsboard/server/dao/util/mapping/AbstractEntityInfosConverter.java new file mode 100644 index 0000000000..b829060f54 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/util/mapping/AbstractEntityInfosConverter.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util.mapping; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.AttributeConverter; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.EntityType; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public abstract class AbstractEntityInfosConverter implements AttributeConverter, String> { + + protected abstract EntityType getEntityType(); + + @Override + public String convertToDatabaseColumn(List attribute) { + throw new IllegalArgumentException("Not implemented!"); + } + + @Override + public List convertToEntityAttribute(String s) { + try { + JsonNode node = JacksonUtil.fromBytes(s.getBytes(StandardCharsets.UTF_8)); + if (node.isArray()) { + List entities = new ArrayList<>(); + for (int i = 0; i < node.size(); i++) { + JsonNode row = node.get(i); + UUID id = null; + String name = null; + JsonNode idNode = row.get("id"); + JsonNode nameNode = row.get("name"); + if (idNode != null && nameNode != null) { + try { + id = UUID.fromString(idNode.asText()); + } catch (Exception ignored) {} + name = nameNode.asText(); + } + if (id != null && name != null) { + entities.add(new EntityInfo(id, EntityType.WIDGETS_BUNDLE.name(), name)); + } + } + return entities; + } else { + return Collections.emptyList(); + } + } catch (Exception ex) { + String exception = String.format("Failed to convert String to %s list: %s", getEntityType(), ex.getMessage()); + throw new RuntimeException(exception, ex); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/mapping/WidgetBundleEntityInfosConverter.java b/dao/src/main/java/org/thingsboard/server/dao/util/mapping/WidgetBundleEntityInfosConverter.java new file mode 100644 index 0000000000..143de09818 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/util/mapping/WidgetBundleEntityInfosConverter.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util.mapping; + +import jakarta.persistence.Converter; +import org.thingsboard.server.common.data.EntityType; + +@Converter +public class WidgetBundleEntityInfosConverter extends AbstractEntityInfosConverter { + + @Override + protected EntityType getEntityType() { + return EntityType.WIDGETS_BUNDLE; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java index d5876bbbd9..e61976b6dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java @@ -56,6 +56,8 @@ public interface WidgetTypeDao extends Dao, ExportableEntityD boolean existsByTenantIdAndId(TenantId tenantId, UUID widgetTypeId); + WidgetTypeInfo findWidgetTypeInfoById(TenantId tenantId, UUID widgetTypeId); + PageData findSystemWidgetTypes(WidgetTypeFilter widgetTypeFilter, PageLink pageLink); PageData findAllTenantWidgetTypesByTenantId(WidgetTypeFilter widgetTypeFilter, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java index a1e241073f..1b0ecb33a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java @@ -87,6 +87,13 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { return widgetTypeDao.findById(tenantId, widgetTypeId.getId()); } + @Override + public WidgetTypeInfo findWidgetTypeInfoById(TenantId tenantId, WidgetTypeId widgetTypeId) { + log.trace("Executing findWidgetTypeInfoById [{}]", widgetTypeId); + Validator.validateId(widgetTypeId, id -> "Incorrect widgetTypeId " + id); + return widgetTypeDao.findWidgetTypeInfoById(tenantId, widgetTypeId.getId()); + } + @Override public boolean widgetTypeExistsByTenantIdAndWidgetTypeId(TenantId tenantId, WidgetTypeId widgetTypeId) { log.trace("Executing widgetTypeExistsByTenantIdAndWidgetTypeId, tenantId [{}], widgetTypeId [{}]", tenantId, widgetTypeId); diff --git a/dao/src/main/resources/sql/schema-views-and-functions.sql b/dao/src/main/resources/sql/schema-views-and-functions.sql index 31219ced18..b394742bcd 100644 --- a/dao/src/main/resources/sql/schema-views-and-functions.sql +++ b/dao/src/main/resources/sql/schema-views-and-functions.sql @@ -288,8 +288,15 @@ $$; DROP VIEW IF EXISTS widget_type_info_view CASCADE; CREATE OR REPLACE VIEW widget_type_info_view AS -SELECT t.* - , COALESCE((t.descriptor::json->>'type')::text, '') as widget_type +SELECT t.*, + COALESCE((t.descriptor::json->>'type')::text, '') as widget_type, + array_to_json(ARRAY( + SELECT json_build_object('id', wb.widgets_bundle_id, 'name', b.title) + FROM widgets_bundle_widget wb + JOIN widgets_bundle b ON wb.widgets_bundle_id = b.id + WHERE wb.widget_type_id = t.id + ORDER BY b.title + )) AS bundles FROM widget_type t; CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid uuid, From fb0b80a35321710f18a496e46f58f2f8a02699e8 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 13 Feb 2025 17:02:46 +0200 Subject: [PATCH 283/416] Calculated field adjustments --- .../calculated-fields-table-config.ts | 2 +- .../calculated-field-dialog.component.html | 24 ++++++++++++------- .../entity/entities-table.component.ts | 2 +- .../components/event/event-table-config.ts | 13 +++++----- .../entity/entities-table-config.models.ts | 6 ++++- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index ce47292397..f1e549d75f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -280,7 +280,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { if (openCalculatedFieldEdit) { - this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + this.editCalculatedField({ entityId: this.entityId, ...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) } }), ); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 6ff0977178..8bd3b5db3f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -100,12 +100,22 @@ [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" helpId="calculated-field/expression_fn" - /> + > + +

        @@ -122,18 +132,16 @@ } - @if (outputFormGroup.get('type').value === OutputType.Attribute) { + @if (outputFormGroup.get('type').value === OutputType.Attribute && data.entityId.entityType === EntityType.DEVICE) { {{ 'calculated-fields.attribute-scope' | translate }} {{ 'calculated-fields.server-attributes' | translate }} - @if (data.entityId.entityType === EntityType.DEVICE) { - - {{ 'calculated-fields.shared-attributes' | translate }} - - } + + {{ 'calculated-fields.shared-attributes' | translate }} + } diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index 4f6f34fcce..e472ad5389 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -689,7 +689,7 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa } cellTooltip(entity: BaseData, column: EntityColumn>, row: number) { - if (column instanceof EntityTableColumn) { + if (column instanceof EntityTableColumn || column instanceof EntityLinkTableColumn) { const col = this.entitiesTableConfig.columns.indexOf(column); const index = row * this.entitiesTableConfig.columns.length + col; let res = this.cellTooltipCache[index]; diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index b32c5f0050..a2ccaa05da 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -18,6 +18,7 @@ import { CellActionDescriptorType, DateEntityTableColumn, EntityActionTableColumn, + EntityLinkTableColumn, EntityTableColumn, EntityTableConfig } from '@home/models/entity/entities-table-config.models'; @@ -29,7 +30,7 @@ import { MatDialog } from '@angular/material/dialog'; import { EntityId } from '@shared/models/id/entity-id'; import { EventService } from '@app/core/http/event.service'; import { EventTableHeaderComponent } from '@home/components/event/event-table-header.component'; -import { EntityTypeResource } from '@shared/models/entity-type.models'; +import { EntityType, EntityTypeResource } from '@shared/models/entity-type.models'; import { fromEvent, Observable } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { Direction } from '@shared/models/page/sort-order'; @@ -39,7 +40,7 @@ import { EventContentDialogComponent, EventContentDialogData } from '@home/components/event/event-content-dialog.component'; -import { isEqual, sortObjectKeys } from '@core/utils'; +import { getEntityDetailsPageURL, isEqual, sortObjectKeys } from '@core/utils'; import { DAY, historyInterval, MINUTE } from '@shared/models/time/time.models'; import { Overlay, OverlayConfig, OverlayRef } from '@angular/cdk/overlay'; import { ChangeDetectorRef, EventEmitter, Injector, StaticProvider, ViewContainerRef } from '@angular/core'; @@ -359,13 +360,13 @@ export class EventTableConfig extends EntityTableConfig { this.columns[0].width = '80px'; this.columns[1].width = '100px'; this.columns.push( - new EntityTableColumn('entityId', 'event.entity-id', '100px', + new EntityLinkTableColumn('entityId', 'event.entity-id', '100px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, + (entity) => getEntityDetailsPageURL(entity.body.entityId, entity.body.entityType as EntityType), () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), - () => undefined, - false, + (entity) => entity.body.entityId, { name: this.translate.instant('event.copy-entity-id'), icon: 'content_paste', @@ -384,7 +385,7 @@ export class EventTableConfig extends EntityTableConfig { () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), - () => undefined, + (entity) => entity.body.msgId, false, { name: this.translate.instant('event.copy-message-id'), diff --git a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts index dbdcdcd61e..6c70a11326 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts @@ -121,7 +121,11 @@ export class EntityLinkTableColumn> extends BaseEntity public width: string = '0px', public cellContentFunction: CellContentFunction = (entity, property) => entity[property] ? entity[property] : '', public entityURL: (entity) => string, - public sortable: boolean = true) { + public cellStyleFunction: CellStyleFunction = () => ({}), + public sortable: boolean = true, + public headerCellStyleFunction: HeaderCellStyleFunction = () => ({}), + public cellTooltipFunction: CellTooltipFunction = () => undefined, + public actionCell: CellActionDescriptor = null) { super('link', key, title, width, sortable); } } From 5aacd04e151bb7184dc024c0a6e79c47998b924e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 13 Feb 2025 17:11:11 +0200 Subject: [PATCH 284/416] Moved calculated fields in device profile tabs --- .../dialog/calculated-field-dialog.component.html | 3 ++- .../device-profile/device-profile-tabs.component.html | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 8bd3b5db3f..2f707c0a2f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -132,7 +132,8 @@ } - @if (outputFormGroup.get('type').value === OutputType.Attribute && data.entityId.entityType === EntityType.DEVICE) { + @if (outputFormGroup.get('type').value === OutputType.Attribute + && (data.entityId.entityType === EntityType.DEVICE || data.entityId.entityType === EntityType.DEVICE_PROFILE)) { {{ 'calculated-fields.attribute-scope' | translate }} diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index 37cfafa9aa..9a3c34c070 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -40,6 +40,10 @@
        + + + - - From 734064d8b1cc12fb4a222d9feae68b09b77b5816 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 13 Feb 2025 17:44:04 +0200 Subject: [PATCH 285/416] Switched link column config order --- .../src/app/modules/home/components/event/event-table-config.ts | 2 +- .../modules/home/models/entity/entities-table-config.models.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index a2ccaa05da..1e9717d5a8 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -363,9 +363,9 @@ export class EventTableConfig extends EntityTableConfig { new EntityLinkTableColumn('entityId', 'event.entity-id', '100px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, (entity) => getEntityDetailsPageURL(entity.body.entityId, entity.body.entityType as EntityType), - () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), + () => ({padding: '0 12px 0 0'}), (entity) => entity.body.entityId, { name: this.translate.instant('event.copy-entity-id'), diff --git a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts index 6c70a11326..49583e90dc 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts @@ -121,8 +121,8 @@ export class EntityLinkTableColumn> extends BaseEntity public width: string = '0px', public cellContentFunction: CellContentFunction = (entity, property) => entity[property] ? entity[property] : '', public entityURL: (entity) => string, - public cellStyleFunction: CellStyleFunction = () => ({}), public sortable: boolean = true, + public cellStyleFunction: CellStyleFunction = () => ({}), public headerCellStyleFunction: HeaderCellStyleFunction = () => ({}), public cellTooltipFunction: CellTooltipFunction = () => undefined, public actionCell: CellActionDescriptor = null) { From fa304180e6afdf820f1701c858182dc4095a660e Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 13 Feb 2025 22:21:06 +0100 Subject: [PATCH 286/416] added tests and refactoring due to comments --- .../controller/TbResourceController.java | 2 +- .../resource/DefaultTbImageService.java | 2 +- .../resource/DefaultTbResourceService.java | 9 +- .../service/resource/TbResourceService.java | 13 ++- .../controller/AbstractNotifyEntityTest.java | 7 +- .../controller/TbResourceControllerTest.java | 74 ++++++++-------- .../sql/BaseTbResourceServiceTest.java | 88 ++++++++++--------- 7 files changed, 109 insertions(+), 86 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java b/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java index 0d4b988771..2f7128f97c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java @@ -217,7 +217,7 @@ public class TbResourceController extends BaseController { @RequestBody TbResource resource) throws Exception { resource.setTenantId(getTenantId()); checkEntity(resource.getId(), resource, Resource.TB_RESOURCE); - return new TbResourceInfo(tbResourceService.save(resource, getCurrentUser())); + return tbResourceService.save(resource, getCurrentUser()); } @ApiOperation(value = "Get Resource Infos (getResources)", diff --git a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java index 63fe2108f3..a1950b5068 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java @@ -126,7 +126,7 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb return savedImage; } catch (Exception e) { image.setData(null); - logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.TB_RESOURCE), image, actionType, user, e); + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.TB_RESOURCE), new TbResourceInfo(image), actionType, user, e); throw e; } } diff --git a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java index 5ff589fd2d..f5ab939635 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java @@ -67,7 +67,7 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements private final AccessControlService accessControlService; @Override - public TbResource save(TbResource resource, SecurityUser user) throws ThingsboardException { + public TbResourceInfo save(TbResource resource, SecurityUser user) throws ThingsboardException { if (resource.getResourceType() == ResourceType.IMAGE) { throw new IllegalArgumentException("Image resource type is not supported"); } @@ -79,12 +79,11 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements } else if (resource.getResourceKey() == null) { resource.setResourceKey(resource.getFileName()); } - TbResource savedResource = resourceService.saveResource(resource); - logEntityActionService.logEntityAction(tenantId, savedResource.getId(), new TbResourceInfo(savedResource), actionType, user); + TbResourceInfo savedResource = new TbResourceInfo(resourceService.saveResource(resource)); + logEntityActionService.logEntityAction(tenantId, savedResource.getId(), savedResource, actionType, user); return savedResource; } catch (Exception e) { - logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.TB_RESOURCE), - resource, actionType, user, e); + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.TB_RESOURCE), new TbResourceInfo(resource), actionType, user, e); throw e; } } diff --git a/application/src/main/java/org/thingsboard/server/service/resource/TbResourceService.java b/application/src/main/java/org/thingsboard/server/service/resource/TbResourceService.java index 0de2cded00..20e6c81a7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/TbResourceService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/TbResourceService.java @@ -18,17 +18,26 @@ package org.thingsboard.server.service.resource; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.ResourceExportData; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.lwm2m.LwM2mObject; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; -import org.thingsboard.server.service.entitiy.SimpleTbEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.List; -public interface TbResourceService extends SimpleTbEntityService { +public interface TbResourceService { + + default TbResourceInfo save(TbResource entity) throws Exception { + return save(entity, null); + } + + TbResourceInfo save(TbResource entity, SecurityUser user) throws Exception; + + void delete(TbResource entity, User user); List findLwM2mObject(TenantId tenantId, String sortOrder, diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index 238203cdb0..f0e8832116 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -629,8 +629,11 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { private String entityClassToEntityTypeName(HasName entity) { String entityType = entityClassToString(entity); - return "SAVE_OTA_PACKAGE_INFO_REQUEST".equals(entityType) || "OTA_PACKAGE_INFO".equals(entityType) ? - EntityType.OTA_PACKAGE.name().toUpperCase(Locale.ENGLISH) : entityType; + return switch (entityType) { + case "SAVE_OTA_PACKAGE_INFO_REQUEST", "OTA_PACKAGE_INFO" -> "OTA_PACKAGE"; + case "TB_RESOURCE_INFO" -> "TB_RESOURCE"; + default -> entityType; + }; } private String entityClassToString(HasName entity) { diff --git a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java index 23f5eadab7..9f2baaa1cd 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java @@ -104,7 +104,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setFileName(DEFAULT_FILE_NAME); resource.setEncodedData(TEST_DATA); - TbResource savedResource = save(resource); + TbResourceInfo savedResource = save(resource); testNotifyEntityAllOneTimeLogEntityActionEntityEqClass(savedResource, savedResource.getId(), savedResource.getId(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), @@ -119,15 +119,15 @@ public class TbResourceControllerTest extends AbstractControllerTest { Assert.assertEquals(DEFAULT_FILE_NAME, savedResource.getResourceKey()); Assert.assertArrayEquals(resource.getData(), download(savedResource.getId())); - savedResource.setTitle("My new resource"); - savedResource.setData(null); + TbResource foundResource = doGet("/api/resource/" + savedResource.getId().getId().toString(), TbResource.class); + foundResource.setTitle("My new resource"); + foundResource.setData(null); - save(savedResource); + savedResource = save(foundResource); - TbResource foundResource = doGet("/api/resource/" + savedResource.getId().getId().toString(), TbResource.class); Assert.assertEquals(foundResource.getTitle(), savedResource.getTitle()); - testNotifyEntityAllOneTimeLogEntityActionEntityEqClass(foundResource, foundResource.getId(), foundResource.getId(), + testNotifyEntityAllOneTimeLogEntityActionEntityEqClass(savedResource, savedResource.getId(), savedResource.getId(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.UPDATED, ActionType.UPDATED); } @@ -147,7 +147,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(resource, savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(new TbResourceInfo(resource), savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @@ -159,7 +159,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setFileName(DEFAULT_FILE_NAME); resource.setEncodedData(TEST_DATA); - TbResource savedResource = save(resource); + TbResourceInfo savedResource = save(resource); loginDifferentTenant(); @@ -188,7 +188,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setFileName(DEFAULT_FILE_NAME); resource.setEncodedData(TEST_DATA); - TbResource savedResource = save(resource); + TbResourceInfo savedResource = save(resource); TbResource foundResource = doGet("/api/resource/" + savedResource.getUuidId(), TbResource.class); Assert.assertNotNull(foundResource); @@ -204,7 +204,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setFileName(DEFAULT_FILE_NAME); resource.setEncodedData(TEST_DATA); - TbResource savedResource = save(resource); + TbResourceInfo savedResource = save(resource); Mockito.reset(tbClusterService, auditLogService); String resourceIdStr = savedResource.getId().getId().toString(); @@ -229,7 +229,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setFileName(DEFAULT_FILE_NAME); resource.setEncodedData(TEST_DATA); - TbResource savedResource = save(resource); + TbResourceInfo savedResource = save(resource); Mockito.reset(tbClusterService, auditLogService); String resourceIdStr = savedResource.getId().getId().toString(); @@ -274,7 +274,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { } } while (pageData.hasNext()); - testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new TbResource(), new TbResource(), + testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new TbResourceInfo(), new TbResourceInfo(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, cntEntity, cntEntity, cntEntity); @@ -322,7 +322,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { } } while (pageData.hasNext()); - testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new TbResource(), new TbResource(), + testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new TbResourceInfo(), new TbResourceInfo(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, jksCntEntity + lwm2mCntEntity, jksCntEntity + lwm2mCntEntity, jksCntEntity + lwm2mCntEntity); @@ -414,7 +414,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setResourceType(ResourceType.PKCS_12); resource.setFileName(i + DEFAULT_FILE_NAME_2); resource.setEncodedData(TEST_DATA); - TbResource saved = save(resource); + TbResourceInfo saved = save(resource); lwm2mesources.add(saved); } @@ -536,7 +536,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setFileName(JS_TEST_FILE_NAME); resource.setEncodedData(TEST_DATA); - TbResource savedResource = save(resource); + TbResourceInfo savedResource = save(resource); testNotifyEntityAllOneTimeLogEntityActionEntityEqClass(savedResource, savedResource.getId(), savedResource.getId(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), @@ -567,7 +567,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setFileName(JS_TEST_FILE_NAME); resource.setEncodedData(TEST_DATA); - TbResource savedResource = save(resource); + TbResourceInfo savedResource = save(resource); //download as public customer Device device = new Device(); @@ -611,7 +611,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setFileName(JS_TEST_FILE_NAME); resource.setEncodedData(TEST_DATA); - TbResource savedResource = save(resource); + TbResourceInfo savedResource = save(resource); loginDifferentTenant(); loginDifferentTenantCustomer(); @@ -626,17 +626,19 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setTitle("My resource"); resource.setFileName("3.pks"); resource.setEncodedData(TEST_DATA); - TbResource savedResource = save(resource); + TbResourceInfo savedResource = save(resource); resource.setEtag(savedResource.getEtag()); - savedResource.setEncodedData(TEST_DATA); - doPost("/api/resource", savedResource) + TbResource foundResource = doGet("/api/resource/" + savedResource.getUuidId(), TbResource.class); + + foundResource.setEncodedData(TEST_DATA); + doPost("/api/resource", foundResource) .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString("can't be updated"))); - savedResource.setData(null); - savedResource.setTitle("Updated resource"); - savedResource = doPost("/api/resource", savedResource, TbResource.class); + foundResource.setData(null); + foundResource.setTitle("Updated resource"); + savedResource = doPost("/api/resource", foundResource, TbResource.class); assertThat(savedResource.getTitle()).isEqualTo("Updated resource"); assertThat(savedResource.getFileName()).isEqualTo(resource.getFileName()); assertThat(savedResource.getEtag()).isEqualTo(resource.getEtag()); @@ -650,14 +652,16 @@ public class TbResourceControllerTest extends AbstractControllerTest { resource.setTitle("My resource"); resource.setFileName("module.js"); resource.setEncodedData(TEST_DATA); - TbResource savedResource = save(resource); - resource.setEtag(savedResource.getEtag()); + TbResourceInfo savedResource = save(resource); + + TbResource foundResource = doGet("/api/resource/" + savedResource.getUuidId(), TbResource.class); + resource.setEtag(foundResource.getEtag()); String newData = Base64.getEncoder().encodeToString(new byte[]{1, 2, 3}); - savedResource.setEncodedData(newData); - savedResource.setFileName("new-module.js"); - savedResource.setTitle("Updated title"); - savedResource = save(savedResource); + foundResource.setEncodedData(newData); + foundResource.setFileName("new-module.js"); + foundResource.setTitle("Updated title"); + savedResource = save(foundResource); assertThat(savedResource.getTitle()).isEqualTo("Updated title"); assertThat(savedResource.getFileName()).isEqualTo("new-module.js"); @@ -669,7 +673,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { public void testGetLwm2mListObjectsPage() throws Exception { loginTenantAdmin(); - List resources = loadLwm2mResources(); + List resources = loadLwm2mResources(); List objects = doGetTyped("/api/resource/lwm2m/page?pageSize=100&page=0", new TypeReference<>() {}); @@ -683,7 +687,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { public void testGetLwm2mListObjects() throws Exception { loginTenantAdmin(); - List resources = loadLwm2mResources(); + List resources = loadLwm2mResources(); List objects = doGetTyped("/api/resource/lwm2m?sortProperty=id&sortOrder=ASC&objectIds=3_1.2,5_1.2,19_1.1", new TypeReference<>() {}); @@ -693,7 +697,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { removeLoadResources(resources); } - private TbResource save(TbResource tbResource) throws Exception { + private TbResourceInfo save(TbResource tbResource) throws Exception { return doPostWithTypedResponse("/api/resource", tbResource, new TypeReference<>() { }); } @@ -709,10 +713,10 @@ public class TbResourceControllerTest extends AbstractControllerTest { } - private List loadLwm2mResources() throws Exception { + private List loadLwm2mResources() throws Exception { var models = List.of("1", "2", "3", "5", "6", "9", "19", "3303"); - List resources = new ArrayList<>(models.size()); + List resources = new ArrayList<>(models.size()); for (String model : models) { String fileName = model + ".xml"; @@ -728,7 +732,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { return resources; } - private void removeLoadResources(List resources) throws Exception { + private void removeLoadResources(List resources) throws Exception { for (TbResourceInfo resource : resources) { doDelete("/api/resource/" + resource.getId().getId().toString()) .andExpect(status().isOk()); diff --git a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java index d7c4a68195..160e12f6a7 100644 --- a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java @@ -202,7 +202,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { } } - private TbResource createResource(String title, String filename) throws Exception { + private TbResourceInfo createResource(String title, String filename) throws Exception { TbResource resource = new TbResource(); resource.setTenantId(tenantId); resource.setTitle(title); @@ -222,25 +222,29 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { resource.setFileName(DEFAULT_FILE_NAME); resource.setData(TEST_DATA); - TbResource savedResource = tbResourceService.save(resource); + TbResourceInfo savedResource = tbResourceService.save(resource); - Assert.assertNotNull(savedResource); - Assert.assertNotNull(savedResource.getId()); - Assert.assertTrue(savedResource.getCreatedTime() > 0); - assertEquals(resource.getTenantId(), savedResource.getTenantId()); - assertEquals(resource.getTitle(), savedResource.getTitle()); - assertEquals(resource.getResourceKey(), savedResource.getResourceKey()); - assertArrayEquals(TEST_DATA, savedResource.getData()); + TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); + + Assert.assertNotNull(foundResource); + Assert.assertNotNull(foundResource.getId()); + Assert.assertTrue(foundResource.getCreatedTime() > 0); + assertEquals(resource.getTenantId(), foundResource.getTenantId()); + assertEquals(resource.getTitle(), foundResource.getTitle()); + assertEquals(resource.getResourceKey(), foundResource.getResourceKey()); + assertArrayEquals(TEST_DATA, foundResource.getData()); - savedResource.setTitle("My new resource"); - savedResource.setData(null); + String title = "My new resource"; - tbResourceService.save(savedResource); - TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); - assertEquals(foundResource.getTitle(), savedResource.getTitle()); + foundResource.setTitle(title); + foundResource.setData(null); + + tbResourceService.save(foundResource); + foundResource = resourceService.findResourceById(tenantId, foundResource.getId()); + assertEquals(title, foundResource.getTitle()); assertArrayEquals(foundResource.getData(), TEST_DATA); - tbResourceService.delete(savedResource, null); + tbResourceService.delete(foundResource, null); } @Test @@ -251,17 +255,19 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { resource.setFileName("test_model.xml"); resource.setEncodedData(Base64.getEncoder().encodeToString(LWM2M_TEST_MODEL.getBytes())); - TbResource savedResource = tbResourceService.save(resource); + TbResourceInfo savedResource = tbResourceService.save(resource); - Assert.assertNotNull(savedResource); - Assert.assertNotNull(savedResource.getId()); - Assert.assertTrue(savedResource.getCreatedTime() > 0); - assertEquals(resource.getTenantId(), savedResource.getTenantId()); - assertEquals("My first resource id=0 v1.0", savedResource.getTitle()); - assertEquals("0_1.0", savedResource.getResourceKey()); - assertArrayEquals(savedResource.getData(), LWM2M_TEST_MODEL.getBytes()); + TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); - tbResourceService.delete(savedResource, null); + Assert.assertNotNull(foundResource); + Assert.assertNotNull(foundResource.getId()); + Assert.assertTrue(foundResource.getCreatedTime() > 0); + assertEquals(resource.getTenantId(), foundResource.getTenantId()); + assertEquals("My first resource id=0 v1.0", foundResource.getTitle()); + assertEquals("0_1.0", foundResource.getResourceKey()); + assertArrayEquals(foundResource.getData(), LWM2M_TEST_MODEL.getBytes()); + + tbResourceService.delete(foundResource, null); } @Test @@ -271,11 +277,11 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { resource.setTitle("My resource"); resource.setFileName(DEFAULT_FILE_NAME); resource.setData(TEST_DATA); - TbResource savedResource = tbResourceService.save(resource); + TbResourceInfo savedResource = tbResourceService.save(resource); assertEquals(TenantId.SYS_TENANT_ID, savedResource.getTenantId()); - tbResourceService.delete(savedResource, null); +// tbResourceService.delete(savedResource, null); } @Test @@ -286,7 +292,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { resource.setTitle("My resource"); resource.setFileName(DEFAULT_FILE_NAME); resource.setData(TEST_DATA); - resource = tbResourceService.save(resource); + TbResourceInfo savedResource = tbResourceService.save(resource); TbResource resource2 = new TbResource(); resource2.setTenantId(tenantId); @@ -294,11 +300,11 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { resource2.setTitle("My resource"); resource2.setFileName(DEFAULT_FILE_NAME); resource2.setData(TEST_DATA); - resource2 = tbResourceService.save(resource2); + TbResourceInfo savedResource2 = tbResourceService.save(resource2); - assertThat(resource2.getId()).isNotEqualTo(resource.getId()); - assertThat(resource2.getFileName()).isEqualTo("test.jks"); - assertThat(resource2.getResourceKey()).isEqualTo("test_(1).jks"); + assertThat(savedResource2.getId()).isNotEqualTo(savedResource.getId()); + assertThat(savedResource2.getFileName()).isEqualTo("test.jks"); + assertThat(savedResource2.getResourceKey()).isEqualTo("test_(1).jks"); } @Test @@ -348,12 +354,13 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { resource.setTitle("My resource"); resource.setFileName(DEFAULT_FILE_NAME); resource.setData(TEST_DATA); - TbResource savedResource = tbResourceService.save(resource); + TbResourceInfo savedResource = tbResourceService.save(resource); TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); Assert.assertNotNull(foundResource); - assertEquals(savedResource, foundResource); - tbResourceService.delete(savedResource, null); + assertEquals(savedResource, new TbResourceInfo(foundResource)); + assertArrayEquals(TEST_DATA, foundResource.getData()); + tbResourceService.delete(foundResource, null); } @Test @@ -364,12 +371,13 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { resource.setTitle("My resource"); resource.setFileName(DEFAULT_FILE_NAME); resource.setData(TEST_DATA); - TbResource savedResource = tbResourceService.save(resource); + TbResourceInfo savedResource = tbResourceService.save(resource); TbResource foundResource = resourceService.findResourceByTenantIdAndKey(tenantId, savedResource.getResourceType(), savedResource.getResourceKey()); Assert.assertNotNull(foundResource); - assertEquals(savedResource, foundResource); - tbResourceService.delete(savedResource, null); + assertEquals(savedResource, new TbResourceInfo(foundResource)); + assertArrayEquals(TEST_DATA, foundResource.getData()); + tbResourceService.delete(foundResource, null); } @Test @@ -379,11 +387,11 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { resource.setTitle("My resource"); resource.setFileName(DEFAULT_FILE_NAME); resource.setData(TEST_DATA); - TbResource savedResource = tbResourceService.save(resource); + TbResourceInfo savedResource = tbResourceService.save(resource); TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); Assert.assertNotNull(foundResource); - tbResourceService.delete(savedResource, null); + tbResourceService.delete(foundResource, null); foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); Assert.assertNull(foundResource); } @@ -471,7 +479,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { resource.setResourceType(ResourceType.JKS); resource.setFileName(i + DEFAULT_FILE_NAME); resource.setData(TEST_DATA); - resources.add(new TbResourceInfo(tbResourceService.save(resource))); + resources.add(tbResourceService.save(resource)); } List loadedResources = new ArrayList<>(); From ca12f96e2cca1b15d0b5dbe910f595b51399386d Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 12 Feb 2025 12:41:49 +0200 Subject: [PATCH 287/416] WidgetsBundle - fixed toString --- .../WidgetsBundleControllerTest.java | 2 +- .../common/data/widget/WidgetsBundle.java | 39 ++++--------------- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java index 03506af95c..ee2fe1dae0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java @@ -95,7 +95,7 @@ public class WidgetsBundleControllerTest extends AbstractControllerTest { Assert.assertEquals(widgetsBundle.getTitle(), savedWidgetsBundle.getTitle()); savedWidgetsBundle.setTitle("My new widgets bundle"); - doPost("/api/widgetsBundle", savedWidgetsBundle, WidgetsBundle.class); + savedWidgetsBundle = doPost("/api/widgetsBundle", savedWidgetsBundle, WidgetsBundle.class); WidgetsBundle foundWidgetsBundle = doGet("/api/widgetsBundle/" + savedWidgetsBundle.getId().getId().toString(), WidgetsBundle.class); Assert.assertEquals(foundWidgetsBundle.getTitle(), savedWidgetsBundle.getTitle()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java index 005e4ca252..82ba041040 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java @@ -17,9 +17,9 @@ package org.thingsboard.server.common.data.widget; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; +import lombok.ToString; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasImage; @@ -32,58 +32,45 @@ import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.io.Serial; + @Schema +@Data @EqualsAndHashCode(callSuper = true) public class WidgetsBundle extends BaseData implements HasName, HasTenantId, ExportableEntity, HasTitle, HasImage, HasVersion { + @Serial private static final long serialVersionUID = -7627368878362410489L; - @Getter - @Setter @Schema(description = "JSON object with Tenant Id.", accessMode = Schema.AccessMode.READ_ONLY) private TenantId tenantId; @NoXss @Length(fieldName = "alias") - @Getter - @Setter @Schema(description = "Unique alias that is used in widget types as a reference widget bundle", accessMode = Schema.AccessMode.READ_ONLY) private String alias; @NoXss @Length(fieldName = "title") - @Getter - @Setter @Schema(description = "Title used in search and UI", accessMode = Schema.AccessMode.READ_ONLY) private String title; - @Getter - @Setter @Schema(description = "Relative or external image URL. Replaced with image data URL (Base64) in case of relative URL and 'inlineImages' option enabled.", accessMode = Schema.AccessMode.READ_ONLY) + @ToString.Exclude private String image; - @Getter - @Setter @Schema(description = "Whether widgets bundle contains SCADA symbol widget types.", accessMode = Schema.AccessMode.READ_ONLY) private boolean scada; @NoXss @Length(fieldName = "description", max = 1024) - @Getter - @Setter @Schema(description = "Description", accessMode = Schema.AccessMode.READ_ONLY) private String description; - @Getter - @Setter @Schema(description = "Order", accessMode = Schema.AccessMode.READ_ONLY) private Integer order; - @Getter - @Setter private WidgetsBundleId externalId; - @Getter - @Setter private Long version; public WidgetsBundle() { @@ -128,16 +115,4 @@ public class WidgetsBundle extends BaseData implements HasName, public String getName() { return title; } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("WidgetsBundle{"); - sb.append("tenantId=").append(tenantId); - sb.append(", alias='").append(alias).append('\''); - sb.append(", title='").append(title).append('\''); - sb.append(", description='").append(description).append('\''); - sb.append('}'); - return sb.toString(); - } - } From 6d840e3686039d3b7df2f3cd0a34891f39645507 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 14 Feb 2025 10:56:59 +0200 Subject: [PATCH 288/416] Improve AbstractEntityInfosConverter --- .../server/dao/util/mapping/AbstractEntityInfosConverter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/mapping/AbstractEntityInfosConverter.java b/dao/src/main/java/org/thingsboard/server/dao/util/mapping/AbstractEntityInfosConverter.java index b829060f54..98b94d17e2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/mapping/AbstractEntityInfosConverter.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/mapping/AbstractEntityInfosConverter.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.util.mapping; import com.fasterxml.jackson.databind.JsonNode; import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; @@ -27,6 +28,7 @@ import java.util.Collections; import java.util.List; import java.util.UUID; +@Converter public abstract class AbstractEntityInfosConverter implements AttributeConverter, String> { protected abstract EntityType getEntityType(); @@ -55,7 +57,7 @@ public abstract class AbstractEntityInfosConverter implements AttributeConverter name = nameNode.asText(); } if (id != null && name != null) { - entities.add(new EntityInfo(id, EntityType.WIDGETS_BUNDLE.name(), name)); + entities.add(new EntityInfo(id, getEntityType().name(), name)); } } return entities; From 4ed379db23f5ae724334fe45a2d7b57853e16e18 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 14 Feb 2025 12:11:37 +0200 Subject: [PATCH 289/416] Fix JpaWidgetTypeDaoTest --- .../common/data/widget/WidgetTypeInfo.java | 13 ++++++++ .../dao/sql/widget/JpaWidgetTypeDaoTest.java | 33 ++++++++++--------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeInfo.java index 2ea98f0c17..803dcbb945 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeInfo.java @@ -90,4 +90,17 @@ public class WidgetTypeInfo extends BaseWidgetType { this.bundles = Collections.emptyList(); } + public WidgetTypeInfo(WidgetTypeDetails widgetTypeDetails, List bundles) { + super(widgetTypeDetails); + this.image = widgetTypeDetails.getImage(); + this.description = widgetTypeDetails.getDescription(); + this.tags = widgetTypeDetails.getTags(); + if (widgetTypeDetails.getDescriptor() != null && widgetTypeDetails.getDescriptor().has("type")) { + this.widgetType = widgetTypeDetails.getDescriptor().get("type").asText(); + } else { + this.widgetType = ""; + } + this.bundles = bundles; + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDaoTest.java index 8010eab4cd..ae73b65581 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDaoTest.java @@ -21,6 +21,7 @@ import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetsBundleId; @@ -51,9 +52,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -/** - * Created by Valerii Sosliuk on 4/30/2017. - */ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { // given search text should find a widget with tags, when searching by tags @@ -78,7 +76,7 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { final String BUNDLE_ALIAS = "BUNDLE_ALIAS"; final int WIDGET_TYPE_COUNT = 3; - List widgetTypeList; + List widgetTypeList; WidgetsBundle widgetsBundle; @Autowired @@ -107,7 +105,7 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { widgetTypeList.sort(Comparator.comparing(BaseWidgetType::getName)); } - WidgetTypeDetails createAndSaveWidgetType(TenantId tenantId, int number) { + WidgetTypeInfo createAndSaveWidgetType(TenantId tenantId, int number) { WidgetTypeDetails widgetType = new WidgetTypeDetails(); widgetType.setTenantId(tenantId); widgetType.setName("WIDGET_TYPE_" + number); @@ -119,7 +117,10 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { widgetType.setDescriptor(descriptor); String[] tags = new String[]{"Tag1_"+number, "Tag2_"+number, "TEST_"+number}; widgetType.setTags(tags); - return widgetTypeDao.save(TenantId.SYS_TENANT_ID, widgetType); + WidgetTypeDetails saved = widgetTypeDao.save(TenantId.SYS_TENANT_ID, widgetType); + List bundles = new ArrayList<>(); + bundles.add(new EntityInfo(widgetsBundle.getId(), widgetsBundle.getName())); + return new WidgetTypeInfo(saved, bundles); } WidgetTypeDetails createAndSaveWidgetType(TenantId tenantId, int number, String[] tags) { @@ -139,7 +140,7 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { @After public void tearDown() { widgetsBundleDao.removeById(TenantId.SYS_TENANT_ID, widgetsBundle.getUuidId()); - for (WidgetType widgetType : widgetTypeList) { + for (WidgetTypeInfo widgetType : widgetTypeList) { widgetTypeDao.removeById(TenantId.SYS_TENANT_ID, widgetType.getUuidId()); } } @@ -160,7 +161,7 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { .widgetTypes(Collections.singletonList("static")).build(), new PageLink(1024, 0, "TYPE_DESCRIPTION", new SortOrder("createdTime"))); assertEquals(1, widgetTypes.getData().size()); - assertEquals(new WidgetTypeInfo(widgetTypeList.get(1)), widgetTypes.getData().get(0)); + assertEquals(widgetTypeList.get(1), widgetTypes.getData().get(0)); widgetTypes = widgetTypeDao.findSystemWidgetTypes( WidgetTypeFilter.builder() @@ -170,7 +171,7 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { .widgetTypes(Collections.emptyList()).build(), new PageLink(1024, 0, "hfgfd tag2_2 ghg", new SortOrder("createdTime"))); assertEquals(1, widgetTypes.getData().size()); - assertEquals(new WidgetTypeInfo(widgetTypeList.get(2)), widgetTypes.getData().get(0)); + assertEquals(widgetTypeList.get(2), widgetTypes.getData().get(0)); } @Test @@ -181,7 +182,7 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { Thread.sleep(2); var widgetType = saveWidgetType(TenantId.SYS_TENANT_ID, "widgetName"); sameNameList.add(widgetType); - widgetTypeList.add(widgetType); + widgetTypeList.add(new WidgetTypeInfo(widgetType)); } sameNameList.sort(Comparator.comparing(BaseWidgetType::getName).thenComparing((BaseWidgetType baseWidgetType) -> baseWidgetType.getId().getId())); List expected = sameNameList.stream().map(WidgetTypeInfo::new).collect(Collectors.toList()); @@ -254,12 +255,12 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { public void testFindTenantWidgetTypesByTenantId() { UUID tenantId = Uuids.timeBased(); for (int i = 0; i < WIDGET_TYPE_COUNT; i++) { - var widgetType = createAndSaveWidgetType(new TenantId(tenantId), i); + var widgetType = createAndSaveWidgetType(TenantId.fromUUID(tenantId), i); widgetTypeList.add(widgetType); } PageData widgetTypes = widgetTypeDao.findTenantWidgetTypesByTenantId( WidgetTypeFilter.builder() - .tenantId(new TenantId(tenantId)) + .tenantId(TenantId.fromUUID(tenantId)) .fullSearch(true) .deprecatedFilter(DeprecatedFilter.ALL) .widgetTypes(null).build(), @@ -363,13 +364,13 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { PageData widgetTypes = widgetTypeDao.findWidgetTypesInfosByWidgetsBundleId(TenantId.SYS_TENANT_ID.getId(), widgetsBundle.getUuidId(),true, DeprecatedFilter.ALL, Collections.singletonList("latest"), new PageLink(1024, 0, "TYPE_DESCRIPTION", new SortOrder("createdTime"))); assertEquals(2, widgetTypes.getData().size()); - assertEquals(new WidgetTypeInfo(widgetTypeList.get(0)), widgetTypes.getData().get(0)); - assertEquals(new WidgetTypeInfo(widgetTypeList.get(2)), widgetTypes.getData().get(1)); + assertEquals(widgetTypeList.get(0), widgetTypes.getData().get(0)); + assertEquals(widgetTypeList.get(2), widgetTypes.getData().get(1)); widgetTypes = widgetTypeDao.findWidgetTypesInfosByWidgetsBundleId(TenantId.SYS_TENANT_ID.getId(), widgetsBundle.getUuidId(), true, DeprecatedFilter.ALL, Collections.emptyList(), new PageLink(1024, 0, "hfgfd TEST_0 ghg", new SortOrder("createdTime"))); assertEquals(1, widgetTypes.getData().size()); - assertEquals(new WidgetTypeInfo(widgetTypeList.get(0)), widgetTypes.getData().get(0)); + assertEquals(widgetTypeList.get(0), widgetTypes.getData().get(0)); } @Test @@ -410,7 +411,7 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { @Test public void testFindByTenantIdAndFqn() { - WidgetType result = widgetTypeList.get(0); + WidgetTypeInfo result = widgetTypeList.get(0); assertNotNull(result); WidgetType widgetType = widgetTypeDao.findByTenantIdAndFqn(TenantId.SYS_TENANT_ID.getId(), "FQN_0"); assertEquals(result.getId(), widgetType.getId()); From b332c8ffa9ec1c6a5fb75cccf044e260b4219585 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 14 Feb 2025 11:50:08 +0100 Subject: [PATCH 290/416] minor test improvement --- .../server/service/resource/sql/BaseTbResourceServiceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java index 160e12f6a7..405423dd03 100644 --- a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java @@ -281,7 +281,8 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { assertEquals(TenantId.SYS_TENANT_ID, savedResource.getTenantId()); -// tbResourceService.delete(savedResource, null); + TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); + tbResourceService.delete(foundResource, null); } @Test From 0e3edb271f7ed702fcb22ab3c499d03de4d8094a Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 14 Feb 2025 13:03:06 +0200 Subject: [PATCH 291/416] Add tests for widget bundle --- .../dao/sql/widget/JpaWidgetTypeDaoTest.java | 94 ++++++++++++++++--- 1 file changed, 82 insertions(+), 12 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDaoTest.java index ae73b65581..693bf86d91 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDaoTest.java @@ -50,7 +50,9 @@ import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { @@ -115,7 +117,7 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { var descriptor = JacksonUtil.newObjectNode(); descriptor.put("type", number % 2 == 0 ? "latest" : "static"); widgetType.setDescriptor(descriptor); - String[] tags = new String[]{"Tag1_"+number, "Tag2_"+number, "TEST_"+number}; + String[] tags = new String[]{"Tag1_" + number, "Tag2_" + number, "TEST_" + number}; widgetType.setTags(tags); WidgetTypeDetails saved = widgetTypeDao.save(TenantId.SYS_TENANT_ID, widgetType); List bundles = new ArrayList<>(); @@ -189,10 +191,10 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { PageData widgetTypesFirstPage = widgetTypeDao.findSystemWidgetTypes( WidgetTypeFilter.builder() - .tenantId(TenantId.SYS_TENANT_ID) - .fullSearch(true) - .deprecatedFilter(DeprecatedFilter.ALL) - .widgetTypes(Collections.singletonList("static")).build(), + .tenantId(TenantId.SYS_TENANT_ID) + .fullSearch(true) + .deprecatedFilter(DeprecatedFilter.ALL) + .widgetTypes(Collections.singletonList("static")).build(), new PageLink(10, 0, null, new SortOrder("name"))); assertEquals(10, widgetTypesFirstPage.getData().size()); assertThat(widgetTypesFirstPage.getData()).containsExactlyElementsOf(expected.subList(0, 10)); @@ -217,10 +219,10 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { PageData widgetTypes = widgetTypeDao.findSystemWidgetTypes( WidgetTypeFilter.builder() - .tenantId(TenantId.SYS_TENANT_ID) - .fullSearch(true) - .deprecatedFilter(DeprecatedFilter.ALL) - .widgetTypes(null).build(), + .tenantId(TenantId.SYS_TENANT_ID) + .fullSearch(true) + .deprecatedFilter(DeprecatedFilter.ALL) + .widgetTypes(null).build(), new PageLink(10, 0, searchText) ); @@ -361,7 +363,7 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { @Test public void testFindByWidgetTypeInfosByBundleId() { - PageData widgetTypes = widgetTypeDao.findWidgetTypesInfosByWidgetsBundleId(TenantId.SYS_TENANT_ID.getId(), widgetsBundle.getUuidId(),true, DeprecatedFilter.ALL, Collections.singletonList("latest"), + PageData widgetTypes = widgetTypeDao.findWidgetTypesInfosByWidgetsBundleId(TenantId.SYS_TENANT_ID.getId(), widgetsBundle.getUuidId(), true, DeprecatedFilter.ALL, Collections.singletonList("latest"), new PageLink(1024, 0, "TYPE_DESCRIPTION", new SortOrder("createdTime"))); assertEquals(2, widgetTypes.getData().size()); assertEquals(widgetTypeList.get(0), widgetTypes.getData().get(0)); @@ -427,7 +429,7 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { @Test public void testFindByImageLink() { - TenantId tenantId = new TenantId(UUID.randomUUID()); + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); WidgetTypeDetails details = createAndSaveWidgetType(tenantId, 0, new String[]{"a"}); details.setDescriptor(JacksonUtil.newObjectNode().put("bg", "/image/tenant/widget.png")); widgetTypeDao.save(tenantId, details); @@ -438,14 +440,82 @@ public class JpaWidgetTypeDaoTest extends AbstractJpaDaoTest { widgetTypeDao.removeById(tenantId, details.getUuidId()); } + @Test + public void testFindWidgetTypesWithBundles() { + PageData widgetTypes = widgetTypeDao.findWidgetTypesInfosByWidgetsBundleId( + TenantId.SYS_TENANT_ID.getId(), + widgetsBundle.getUuidId(), + true, + DeprecatedFilter.ALL, + Collections.singletonList("latest"), + new PageLink(1024, 0, "TYPE_DESCRIPTION", new SortOrder("createdTime")) + ); + + assertEquals(2, widgetTypes.getData().size()); + for (var widgetType : widgetTypes.getData()) { + assertFalse("Bundles should not be empty", widgetType.getBundles().isEmpty()); + assertEquals(BUNDLE_ALIAS, widgetType.getBundles().get(0).getName()); + } + } + + @Test + public void testAddWidgetTypeToNewBundleAndVerifyBundles() { + String newBundleTitle = "New Bundle Title"; + WidgetsBundle newWidgetsBundle = new WidgetsBundle(); + newWidgetsBundle.setAlias("NewBundle"); + newWidgetsBundle.setTitle(newBundleTitle); + newWidgetsBundle.setId(new WidgetsBundleId(UUID.randomUUID())); + newWidgetsBundle.setTenantId(TenantId.SYS_TENANT_ID); + newWidgetsBundle = widgetsBundleDao.save(TenantId.SYS_TENANT_ID, newWidgetsBundle); + + for (int i = 0; i < widgetTypeList.size(); i++) { + WidgetTypeInfo widgetType = widgetTypeList.get(i); + widgetTypeDao.saveWidgetsBundleWidget(new WidgetsBundleWidget(newWidgetsBundle.getId(), widgetType.getId(), i)); + } + + PageData widgetTypes = widgetTypeDao.findWidgetTypesInfosByWidgetsBundleId( + TenantId.SYS_TENANT_ID.getId(), + newWidgetsBundle.getUuidId(), + true, + DeprecatedFilter.ALL, + Collections.singletonList("latest"), + new PageLink(1024, 0, "TYPE_DESCRIPTION", new SortOrder("createdTime")) + ); + + assertEquals(2, widgetTypes.getData().size()); + WidgetTypeInfo widgetTypeInfo1 = widgetTypes.getData().get(0); + WidgetTypeInfo widgetTypeInfo2 = widgetTypes.getData().get(1); + + assertEquals(2, widgetTypeInfo1.getBundles().size()); + assertTrue("Bundles should contain 'BUNDLE_ALIAS'", widgetTypeInfo1.getBundles().stream().anyMatch(bundle -> BUNDLE_ALIAS.equals(bundle.getName()))); + assertTrue("Bundles should contain 'New Bundle Title'", widgetTypeInfo1.getBundles().stream().anyMatch(bundle -> newBundleTitle.equals(bundle.getName()))); + + assertEquals(2, widgetTypeInfo2.getBundles().size()); + assertTrue("Bundles should contain 'BUNDLE_ALIAS'", widgetTypeInfo2.getBundles().stream().anyMatch(bundle -> "BUNDLE_ALIAS".equals(bundle.getName()))); + assertTrue("Bundles should contain 'New Bundle Title'", widgetTypeInfo2.getBundles().stream().anyMatch(bundle -> newBundleTitle.equals(bundle.getName()))); + + // cleanup and verify + widgetsBundleDao.removeById(newWidgetsBundle.getTenantId(), newWidgetsBundle.getUuidId()); + widgetTypes = widgetTypeDao.findWidgetTypesInfosByWidgetsBundleId( + TenantId.SYS_TENANT_ID.getId(), + newWidgetsBundle.getUuidId(), + true, + DeprecatedFilter.ALL, + Collections.singletonList("latest"), + new PageLink(1024, 0, "TYPE_DESCRIPTION", new SortOrder("createdTime")) + ); + widgetTypes.getData().forEach(widgetTypeInfo -> assertEquals(1, widgetTypeInfo.getBundles().size())); + } + private WidgetTypeDetails saveWidgetType(TenantId tenantId, String name) { WidgetTypeDetails widgetType = new WidgetTypeDetails(); widgetType.setTenantId(tenantId); widgetType.setDescription("WIDGET_TYPE_DESCRIPTION" + StringUtils.randomAlphabetic(7)); widgetType.setName(name); var descriptor = JacksonUtil.newObjectNode(); - descriptor.put("type","static"); + descriptor.put("type", "static"); widgetType.setDescriptor(descriptor); return widgetTypeDao.save(TenantId.SYS_TENANT_ID, widgetType); } + } From 53f3da153d03828174a1302ef0c68c451562b8b8 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 14 Feb 2025 15:28:28 +0200 Subject: [PATCH 292/416] Added debug events action from calculated field table --- .../calculated-fields/calculated-fields-table-config.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index f1e549d75f..e7ad697262 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -118,6 +118,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, onAction: (event$, entity) => this.exportCalculatedField(event$, entity), }, + { + name: this.translate.instant('entity-view.events'), + icon: 'history', + isEnabled: () => true, + onAction: (_, entity) => this.openDebugDialog(entity), + }, { name: '', nameFunction: entity => this.getDebugConfigLabel(entity?.debugSettings), From dad0c1fb927750f31a73677b8a367369eb812829 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 14 Feb 2025 16:32:48 +0200 Subject: [PATCH 293/416] Version set to 3.9.1-RC --- application/pom.xml | 2 +- common/actor/pom.xml | 2 +- common/cache/pom.xml | 2 +- common/cluster-api/pom.xml | 2 +- common/coap-server/pom.xml | 2 +- common/dao-api/pom.xml | 2 +- common/data/pom.xml | 2 +- common/edge-api/pom.xml | 2 +- common/message/pom.xml | 2 +- common/pom.xml | 2 +- common/proto/pom.xml | 2 +- common/queue/pom.xml | 2 +- common/script/pom.xml | 2 +- common/script/remote-js-client/pom.xml | 2 +- common/script/script-api/pom.xml | 2 +- common/stats/pom.xml | 2 +- common/transport/coap/pom.xml | 2 +- common/transport/http/pom.xml | 2 +- common/transport/lwm2m/pom.xml | 2 +- common/transport/mqtt/pom.xml | 2 +- common/transport/pom.xml | 2 +- common/transport/snmp/pom.xml | 2 +- common/transport/transport-api/pom.xml | 2 +- common/util/pom.xml | 2 +- common/version-control/pom.xml | 2 +- dao/pom.xml | 2 +- monitoring/pom.xml | 2 +- msa/black-box-tests/pom.xml | 2 +- msa/js-executor/package.json | 2 +- msa/js-executor/pom.xml | 2 +- msa/monitoring/pom.xml | 2 +- msa/pom.xml | 2 +- msa/tb-node/pom.xml | 2 +- msa/tb/pom.xml | 2 +- msa/transport/coap/pom.xml | 2 +- msa/transport/http/pom.xml | 2 +- msa/transport/lwm2m/pom.xml | 2 +- msa/transport/mqtt/pom.xml | 2 +- msa/transport/pom.xml | 2 +- msa/transport/snmp/pom.xml | 2 +- msa/vc-executor-docker/pom.xml | 2 +- msa/vc-executor/pom.xml | 2 +- msa/web-ui/package.json | 2 +- msa/web-ui/pom.xml | 2 +- netty-mqtt/pom.xml | 4 ++-- pom.xml | 2 +- rest-client/pom.xml | 2 +- rule-engine/pom.xml | 2 +- rule-engine/rule-engine-api/pom.xml | 2 +- rule-engine/rule-engine-components/pom.xml | 2 +- tools/pom.xml | 2 +- transport/coap/pom.xml | 2 +- transport/http/pom.xml | 2 +- transport/lwm2m/pom.xml | 2 +- transport/mqtt/pom.xml | 2 +- transport/pom.xml | 2 +- transport/snmp/pom.xml | 2 +- ui-ngx/package.json | 2 +- ui-ngx/pom.xml | 2 +- 59 files changed, 60 insertions(+), 60 deletions(-) diff --git a/application/pom.xml b/application/pom.xml index c2d80d62ac..e13605f292 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard application diff --git a/common/actor/pom.xml b/common/actor/pom.xml index cbbce12336..05e770888c 100644 --- a/common/actor/pom.xml +++ b/common/actor/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/cache/pom.xml b/common/cache/pom.xml index a4f7dd4f7b..d97b41fed4 100644 --- a/common/cache/pom.xml +++ b/common/cache/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/cluster-api/pom.xml b/common/cluster-api/pom.xml index 4bbe65d99f..d789732519 100644 --- a/common/cluster-api/pom.xml +++ b/common/cluster-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/coap-server/pom.xml b/common/coap-server/pom.xml index 32da47ca08..9bd3fc19d0 100644 --- a/common/coap-server/pom.xml +++ b/common/coap-server/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/dao-api/pom.xml b/common/dao-api/pom.xml index 7d48a63903..0a84902178 100644 --- a/common/dao-api/pom.xml +++ b/common/dao-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/data/pom.xml b/common/data/pom.xml index e6eca1aa93..6e3d66413a 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/edge-api/pom.xml b/common/edge-api/pom.xml index 68e647642d..e78dff790d 100644 --- a/common/edge-api/pom.xml +++ b/common/edge-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/message/pom.xml b/common/message/pom.xml index 0614c94bc4..53fbdf97c4 100644 --- a/common/message/pom.xml +++ b/common/message/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/pom.xml b/common/pom.xml index 8bd1904d72..9303be4522 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard common diff --git a/common/proto/pom.xml b/common/proto/pom.xml index fded7298e4..215fe27745 100644 --- a/common/proto/pom.xml +++ b/common/proto/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/queue/pom.xml b/common/queue/pom.xml index fab4ff3989..2349d55379 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/script/pom.xml b/common/script/pom.xml index 1aad1dd39e..d2a7a8bfcf 100644 --- a/common/script/pom.xml +++ b/common/script/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/script/remote-js-client/pom.xml b/common/script/remote-js-client/pom.xml index 38a14cd8f3..8aacb3bc59 100644 --- a/common/script/remote-js-client/pom.xml +++ b/common/script/remote-js-client/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.9.0-RC + 3.9.1-RC script org.thingsboard.common.script diff --git a/common/script/script-api/pom.xml b/common/script/script-api/pom.xml index 4210f1f528..f30adc303a 100644 --- a/common/script/script-api/pom.xml +++ b/common/script/script-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.9.0-RC + 3.9.1-RC script org.thingsboard.common.script diff --git a/common/stats/pom.xml b/common/stats/pom.xml index 231811144a..4a23faf475 100644 --- a/common/stats/pom.xml +++ b/common/stats/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/transport/coap/pom.xml b/common/transport/coap/pom.xml index 0bd4c0d6e3..dc5ecd7905 100644 --- a/common/transport/coap/pom.xml +++ b/common/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.common.transport diff --git a/common/transport/http/pom.xml b/common/transport/http/pom.xml index 5a0dd9e4ff..bfb830d8ee 100644 --- a/common/transport/http/pom.xml +++ b/common/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.common.transport diff --git a/common/transport/lwm2m/pom.xml b/common/transport/lwm2m/pom.xml index ddb9d7a399..d5603a86ca 100644 --- a/common/transport/lwm2m/pom.xml +++ b/common/transport/lwm2m/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.common.transport diff --git a/common/transport/mqtt/pom.xml b/common/transport/mqtt/pom.xml index 45d228f225..c043d88e20 100644 --- a/common/transport/mqtt/pom.xml +++ b/common/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.common.transport diff --git a/common/transport/pom.xml b/common/transport/pom.xml index 6c4310151d..01fa1c3c4a 100644 --- a/common/transport/pom.xml +++ b/common/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/transport/snmp/pom.xml b/common/transport/snmp/pom.xml index a2b9dd7c84..b1a78519d3 100644 --- a/common/transport/snmp/pom.xml +++ b/common/transport/snmp/pom.xml @@ -21,7 +21,7 @@ org.thingsboard.common - 3.9.0-RC + 3.9.1-RC transport diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 8f01cab08b..e05414e77b 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.common.transport diff --git a/common/util/pom.xml b/common/util/pom.xml index 194724b70f..b0543bf7fb 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index 8e3fceff99..16559c2e2c 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC common org.thingsboard.common diff --git a/dao/pom.xml b/dao/pom.xml index c1f5a1f62b..3dbdd086db 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard dao diff --git a/monitoring/pom.xml b/monitoring/pom.xml index e19acb0fe6..3b2f378a17 100644 --- a/monitoring/pom.xml +++ b/monitoring/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index 5563fcc8d8..549d773297 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -21,7 +21,7 @@ org.thingsboard - 3.9.0-RC + 3.9.1-RC msa org.thingsboard.msa diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 0fb241c558..d05bbaf71c 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard-js-executor", "private": true, - "version": "3.9.0", + "version": "3.9.1", "description": "ThingsBoard JavaScript Executor Microservice", "main": "server.ts", "bin": "server.js", diff --git a/msa/js-executor/pom.xml b/msa/js-executor/pom.xml index daea52bfa5..8641384cb0 100644 --- a/msa/js-executor/pom.xml +++ b/msa/js-executor/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC msa org.thingsboard.msa diff --git a/msa/monitoring/pom.xml b/msa/monitoring/pom.xml index 20f8d90695..a58da78c17 100644 --- a/msa/monitoring/pom.xml +++ b/msa/monitoring/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC msa diff --git a/msa/pom.xml b/msa/pom.xml index 057f37a575..2ccf92deb3 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard msa diff --git a/msa/tb-node/pom.xml b/msa/tb-node/pom.xml index 3ca22bcf55..cc93c0a078 100644 --- a/msa/tb-node/pom.xml +++ b/msa/tb-node/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC msa org.thingsboard.msa diff --git a/msa/tb/pom.xml b/msa/tb/pom.xml index 5e79740c8b..dab3a52710 100644 --- a/msa/tb/pom.xml +++ b/msa/tb/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC msa org.thingsboard.msa diff --git a/msa/transport/coap/pom.xml b/msa/transport/coap/pom.xml index 089581f1cb..896915082e 100644 --- a/msa/transport/coap/pom.xml +++ b/msa/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.msa.transport diff --git a/msa/transport/http/pom.xml b/msa/transport/http/pom.xml index 4d9e17df5f..483c40c880 100644 --- a/msa/transport/http/pom.xml +++ b/msa/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.msa.transport diff --git a/msa/transport/lwm2m/pom.xml b/msa/transport/lwm2m/pom.xml index 94b86d035f..db98c0828f 100644 --- a/msa/transport/lwm2m/pom.xml +++ b/msa/transport/lwm2m/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.msa.transport diff --git a/msa/transport/mqtt/pom.xml b/msa/transport/mqtt/pom.xml index 2b637ddd19..26d53f6b2f 100644 --- a/msa/transport/mqtt/pom.xml +++ b/msa/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.msa.transport diff --git a/msa/transport/pom.xml b/msa/transport/pom.xml index 4f7e0ad030..c9473462d7 100644 --- a/msa/transport/pom.xml +++ b/msa/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC msa org.thingsboard.msa diff --git a/msa/transport/snmp/pom.xml b/msa/transport/snmp/pom.xml index 94bc03f29d..7c6f267964 100644 --- a/msa/transport/snmp/pom.xml +++ b/msa/transport/snmp/pom.xml @@ -21,7 +21,7 @@ org.thingsboard.msa transport - 3.9.0-RC + 3.9.1-RC org.thingsboard.msa.transport diff --git a/msa/vc-executor-docker/pom.xml b/msa/vc-executor-docker/pom.xml index f6dc43ea13..8cba9d153b 100644 --- a/msa/vc-executor-docker/pom.xml +++ b/msa/vc-executor-docker/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC msa org.thingsboard.msa diff --git a/msa/vc-executor/pom.xml b/msa/vc-executor/pom.xml index 725c3e40ad..5e86f5dd47 100644 --- a/msa/vc-executor/pom.xml +++ b/msa/vc-executor/pom.xml @@ -21,7 +21,7 @@ org.thingsboard - 3.9.0-RC + 3.9.1-RC msa org.thingsboard.msa diff --git a/msa/web-ui/package.json b/msa/web-ui/package.json index 84b1a4a488..deca43eecd 100644 --- a/msa/web-ui/package.json +++ b/msa/web-ui/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard-web-ui", "private": true, - "version": "3.9.0", + "version": "3.9.1", "description": "ThingsBoard Web UI Microservice", "main": "server.ts", "bin": "server.js", diff --git a/msa/web-ui/pom.xml b/msa/web-ui/pom.xml index c5a900e004..bfbcb81012 100644 --- a/msa/web-ui/pom.xml +++ b/msa/web-ui/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC msa org.thingsboard.msa diff --git a/netty-mqtt/pom.xml b/netty-mqtt/pom.xml index d4cd55629d..34a4a3e81a 100644 --- a/netty-mqtt/pom.xml +++ b/netty-mqtt/pom.xml @@ -19,11 +19,11 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard netty-mqtt - 3.9.0-RC + 3.9.1-RC jar Netty MQTT Client diff --git a/pom.xml b/pom.xml index 4de31c1c19..f39fcc0cc9 100755 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard thingsboard - 3.9.0-RC + 3.9.1-RC pom Thingsboard diff --git a/rest-client/pom.xml b/rest-client/pom.xml index f868acabf5..69cd079e6b 100644 --- a/rest-client/pom.xml +++ b/rest-client/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard rest-client diff --git a/rule-engine/pom.xml b/rule-engine/pom.xml index fbefcd1de9..5ed4ee9329 100644 --- a/rule-engine/pom.xml +++ b/rule-engine/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard rule-engine diff --git a/rule-engine/rule-engine-api/pom.xml b/rule-engine/rule-engine-api/pom.xml index 3ee6496452..4db9a7c4bf 100644 --- a/rule-engine/rule-engine-api/pom.xml +++ b/rule-engine/rule-engine-api/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC rule-engine org.thingsboard.rule-engine diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 33bb15af00..5b48de2224 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC rule-engine org.thingsboard.rule-engine diff --git a/tools/pom.xml b/tools/pom.xml index dce8e1606d..f60289c5f4 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard tools diff --git a/transport/coap/pom.xml b/transport/coap/pom.xml index eef23eae9b..0b147c871a 100644 --- a/transport/coap/pom.xml +++ b/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.transport diff --git a/transport/http/pom.xml b/transport/http/pom.xml index d841c56da7..f8e80627f3 100644 --- a/transport/http/pom.xml +++ b/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.transport diff --git a/transport/lwm2m/pom.xml b/transport/lwm2m/pom.xml index cec30fcbc5..43f61039ed 100644 --- a/transport/lwm2m/pom.xml +++ b/transport/lwm2m/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.transport diff --git a/transport/mqtt/pom.xml b/transport/mqtt/pom.xml index edf3727320..0fea251b03 100644 --- a/transport/mqtt/pom.xml +++ b/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC transport org.thingsboard.transport diff --git a/transport/pom.xml b/transport/pom.xml index d744f7eb2f..e101a84e79 100644 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard transport diff --git a/transport/snmp/pom.xml b/transport/snmp/pom.xml index dcd08c10af..26f68cc598 100644 --- a/transport/snmp/pom.xml +++ b/transport/snmp/pom.xml @@ -21,7 +21,7 @@ org.thingsboard - 3.9.0-RC + 3.9.1-RC transport diff --git a/ui-ngx/package.json b/ui-ngx/package.json index 0eff3bbd8d..0d5e086896 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -1,6 +1,6 @@ { "name": "thingsboard", - "version": "3.9.0", + "version": "3.9.1", "scripts": { "ng": "ng", "start": "node --max_old_space_size=8048 ./node_modules/@angular/cli/bin/ng serve --configuration development --host 0.0.0.0 --open", diff --git a/ui-ngx/pom.xml b/ui-ngx/pom.xml index 1384b1c3e6..b00064e2fd 100644 --- a/ui-ngx/pom.xml +++ b/ui-ngx/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.9.0-RC + 3.9.1-RC thingsboard org.thingsboard From 7911384f375f8e823afd51974a53315598673aa4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 14 Feb 2025 16:42:55 +0200 Subject: [PATCH 294/416] fixed tests, added error without stacktrace and param to system params --- .../server/actors/ActorSystemContext.java | 74 +++++++++++-------- ...alculatedFieldManagerMessageProcessor.java | 14 ++-- .../controller/SystemInfoController.java | 11 ++- .../cf/ctx/state/TsRollingArgumentEntry.java | 2 +- .../src/main/resources/thingsboard.yml | 6 ++ .../state/ScriptCalculatedFieldStateTest.java | 55 ++------------ .../state/SimpleCalculatedFieldStateTest.java | 10 +-- .../state/SingleValueArgumentEntryTest.java | 7 +- .../ctx/state/TsRollingArgumentEntryTest.java | 60 +++++++++++---- .../server/common/data/SystemParams.java | 1 + .../data/event/CalculatedFieldDebugEvent.java | 1 - 11 files changed, 125 insertions(+), 116 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index b8344c6c0a..20f61115e6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -43,7 +43,6 @@ import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; @@ -624,6 +623,14 @@ public class ActorSystemContext { @Getter private String debugPerTenantLimitsConfiguration; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + @Getter + private boolean calculatedFieldsDebugPerTenantEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + @Getter + private String calculatedFieldsDebugPerTenantLimitsConfiguration; + @Value("${actors.rpc.submit_strategy:BURST}") @Getter private String rpcSubmitStrategy; @@ -810,39 +817,42 @@ public class ActorSystemContext { } public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { - if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, tenantId)) { - throw new TbRateLimitsException(EntityType.TENANT); - } - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(calculatedFieldId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(entityId); - if (tbMsgId != null) { - eventBuilder.msgId(tbMsgId); - } - if (tbMsgType != null) { - eventBuilder.msgType(tbMsgType.name()); - } - if (arguments != null) { - eventBuilder.arguments(JacksonUtil.toString( - arguments.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) - )); - } - if (result != null) { - eventBuilder.result(result); - } - if (error != null) { - eventBuilder.error(toString(error)); + if (calculatedFieldsDebugPerTenantEnabled) { + if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, calculatedFieldsDebugPerTenantLimitsConfiguration)) { + log.trace("[{}] Calculated field debug event limits exceeded!", tenantId); + throw new TbRateLimitsException("Failed to persist calculated field debug event due to rate limits!"); } + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(calculatedFieldId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); + } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString( + arguments.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) + )); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(error.getMessage()); + } - ListenableFuture future = eventService.saveAsync(eventBuilder.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 898e339426..c3aa378524 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -262,6 +262,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + newCfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(newCf)) { + systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); @@ -286,13 +293,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { - try { - newCfCtx.init(); - } catch (Exception e) { - if (DebugModeUtil.isDebugAllAvailable(newCf)) { - systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); - } - } initCf(newCfCtx, callback, stateChanges); } else { callback.onSuccess(); diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index 8b958df68b..d630c5a22a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.SystemParams; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.mobile.qrCodeSettings.QRCodeConfig; +import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsType; @@ -80,6 +80,12 @@ public class SystemInfoController extends BaseController { @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") private String ruleChainDebugPerTenantLimitsConfiguration; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + private boolean calculatedFieldDebugPerTenantLimitsEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + private String calculatedFieldDebugPerTenantLimitsConfiguration; + @Autowired(required = false) private BuildProperties buildProperties; @@ -155,6 +161,9 @@ public class SystemInfoController extends BaseController { if (ruleChainDebugPerTenantLimitsEnabled) { systemParams.setRuleChainDebugPerTenantLimitsConfiguration(ruleChainDebugPerTenantLimitsConfiguration); } + if (calculatedFieldDebugPerTenantLimitsEnabled) { + systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(calculatedFieldDebugPerTenantLimitsConfiguration); + } } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) .map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index cdad1145e8..1ecf38746e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -43,9 +43,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords = new TreeMap<>(); public TsRollingArgumentEntry(List kvEntries, int limit, long timeWindow) { - kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); this.limit = limit; this.timeWindow = timeWindow; + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); } public TsRollingArgumentEntry(TreeMap tsRecords, int limit, long timeWindow) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 30a3e51d8c..35e8c0e970 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -510,6 +510,12 @@ actors: js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" + calculated_fields: + debug_mode_rate_limits_per_tenant: + # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant + enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" + # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour + configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" debug: settings: diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 30ad7a6507..0227365ed7 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -57,7 +57,7 @@ public class ScriptCalculatedFieldStateTest { private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); - private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("assetHumidity", 43L), 122L); + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("assetHumidity", 43.0), 122L); private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); private final long ts = System.currentTimeMillis(); @@ -127,52 +127,7 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43L)); - } - - @Test - void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException { - TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - - TreeMap values = new TreeMap<>(); - values.put(ts - 40000, 4.0);// will not be used for calculation - values.put(ts - 45000, 2.0);// will not be used for calculation - values.put(ts - 20, 0.0); - - argumentEntry.setTsRecords(values); - - state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); - - CalculatedFieldResult result = state.performCalculation(ctx).get(); - - assertThat(result).isNotNull(); - Output output = getCalculatedFieldConfig().getOutput(); - assertThat(result.getType()).isEqualTo(output.getType()); - assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); - } - - @Test - void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException { - TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 20, 1000.0);// will not be used - values.put(ts - 18, 0.0); - values.put(ts - 16, 0.0); - values.put(ts - 14, 0.0); - values.put(ts - 12, 0.0); - values.put(ts - 10, 0.0); - argumentEntry.setTsRecords(values); - - state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); - - CalculatedFieldResult result = state.performCalculation(ctx).get(); - - assertThat(result).isNotNull(); - Output output = getCalculatedFieldConfig().getOutput(); - assertThat(result.getType()).isEqualTo(output.getType()); - assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); + assertThat(result.getResultMap()).isEqualTo(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0)); } @Test @@ -189,7 +144,7 @@ public class ScriptCalculatedFieldStateTest { @Test void testIsReadyWhenEmptyEntryPresents() { -// state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry)); + state.arguments = new HashMap<>(Map.of("deviceTemperature", new TsRollingArgumentEntry(5, 30000L), "assetHumidity", assetHumidityArgEntry)); assertThat(state.isReady()).isFalse(); } @@ -235,7 +190,7 @@ public class ScriptCalculatedFieldStateTest { config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); - config.setExpression("var result = 0; foreach(element : deviceTemperature.entrySet()) { result += element.getValue(); } var map = {}; map.put(\"averageDeviceTemperature\", result / deviceTemperature.size()); map.put(\"assetHumidity\", assetHumidity); return map;"); + config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity.value}"); Output output = new Output(); output.setType(OutputType.ATTRIBUTES); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index d87e690478..c451ffcbfc 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -117,10 +117,10 @@ public class SimpleCalculatedFieldStateTest { "key2", key2ArgEntry )); -// Map newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY); -// assertThatThrownBy(() -> state.updateState(newArgs)) -// .isInstanceOf(IllegalArgumentException.class) -// .hasMessage("Rolling argument entry is not supported for simple calculated fields."); + Map newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); + assertThatThrownBy(() -> state.updateState(newArgs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Rolling argument entry is not supported for simple calculated fields."); } @Test @@ -175,7 +175,7 @@ public class SimpleCalculatedFieldStateTest { "key1", key1ArgEntry, "key2", key2ArgEntry )); -// state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY); + state.getArguments().put("key3", new SingleValueArgumentEntry()); assertThat(state.isReady()).isFalse(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 57e4655d95..2514b9f34e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test; import org.thingsboard.server.common.data.kv.LongDataEntry; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class SingleValueArgumentEntryTest { @@ -39,9 +40,9 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWhenRollingEntryPassed() { -// assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY)) -// .isInstanceOf(IllegalArgumentException.class) -// .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index e4d5a59d03..75aca61825 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; @@ -40,7 +39,7 @@ public class TsRollingArgumentEntryTest { values.put(ts - 30, 12.0); values.put(ts - 20, 17.0); -// entry = new TsRollingArgumentEntry(values); + entry = new TsRollingArgumentEntry(5, 30000L, values); } @Test @@ -57,18 +56,10 @@ public class TsRollingArgumentEntryTest { assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23.0); } - @Test - void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, new DoubleDataEntry("key", 23.0), 123L); - - assertThat(entry.updateEntry(newEntry)).isFalse(); - } - @Test void testUpdateEntryWhenRollingEntryPassed() { TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); TreeMap values = new TreeMap<>(); - values.put(ts - 20, 16.0); values.put(ts - 10, 7.0); values.put(ts - 5, 1.0); newEntry.setTsRecords(values); @@ -76,11 +67,11 @@ public class TsRollingArgumentEntryTest { assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(5); assertThat(entry.getTsRecords()).isEqualTo(Map.of( - ts - 40, new DoubleDataEntry("key", 10.0), - ts - 30, new DoubleDataEntry("key", 12.0), - ts - 20, new DoubleDataEntry("key", 17.0), - ts - 10, new DoubleDataEntry("key", 7.0), - ts - 5, new DoubleDataEntry("key", 1.0) + ts - 40, 10.0, + ts - 30, 12.0, + ts - 20, 17.0, + ts - 10, 7.0, + ts - 5, 1.0 )); } @@ -90,7 +81,44 @@ public class TsRollingArgumentEntryTest { assertThatThrownBy(() -> entry.updateEntry(newEntry)) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Argument type " + ArgumentEntryType.TS_ROLLING + " only supports numeric values."); + .hasMessage("Time series rolling arguments supports only numeric values."); + } + + @Test + void testUpdateEntryWhenOldTelemetry() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, 4.0);// will not be used for calculation + values.put(ts - 45000, 2.0);// will not be used for calculation + values.put(ts - 5, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(1); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 5, 0.0 + )); + } + + @Test + void testPerformCalculationWhenArgumentsMoreThanLimit() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 1000.0);// will not be used + values.put(ts - 18, 0.0); + values.put(ts - 16, 0.0); + values.put(ts - 14, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(3); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 18, 0.0, + ts - 16, 0.0, + ts - 14, 0.0 + )); } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index abe1932327..53f39977c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -34,4 +34,5 @@ public class SystemParams { boolean mobileQrEnabled; int maxDebugModeDurationMinutes; String ruleChainDebugPerTenantLimitsConfiguration; + String calculatedFieldDebugPerTenantLimitsConfiguration; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java index e0599db358..9b64dd0d40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java @@ -91,5 +91,4 @@ public class CalculatedFieldDebugEvent extends Event { return eventInfo; } - } From 4bd0833eed0df0c694e99abb39ded6d22cc8922e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 14 Feb 2025 18:39:45 +0200 Subject: [PATCH 295/416] Added Calculated field arguments autocomplete --- .../calculated-fields-table-config.ts | 29 +++++-- .../calculated-field-dialog.component.html | 1 + .../calculated-field-dialog.component.ts | 6 ++ ...ed-field-script-test-dialog.component.html | 1 + .../shared/models/calculated-field.models.ts | 79 +++++++++++++++++++ 5 files changed, 110 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index e7ad697262..8f967b68ce 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -36,9 +36,14 @@ import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap, tap } from 'rxjs/operators'; import { + ArgumentType, CalculatedField, + CalculatedFieldArgument, + CalculatedFieldEventArguments, CalculatedFieldDebugDialogData, CalculatedFieldDialogData, + CalculatedFieldRollingValueArgumentAutocomplete, + CalculatedFieldSingleValueArgumentAutocomplete, CalculatedFieldTestScriptDialogData, } from '@shared/models/calculated-field.models'; import { @@ -48,6 +53,7 @@ import { } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; import { isObject } from '@core/utils'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -58,7 +64,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, calculatedField), + action: (calculatedField: CalculatedField) => this.openDebugEventsDialog.call(this, calculatedField), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -122,7 +128,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, - onAction: (_, entity) => this.openDebugDialog(entity), + onAction: (_, entity) => this.openDebugEventsDialog(entity), }, { name: '', @@ -149,7 +155,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog(calculatedField) + action: () => this.openDebugEventsDialog(calculatedField) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -211,14 +217,15 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], @@ -267,7 +274,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } - private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record, openCalculatedFieldEdit = true): Observable { + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: CalculatedFieldEventArguments, openCalculatedFieldEdit = true): Observable { const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) ? argumentsObj[key] : ''; return acc; @@ -279,6 +286,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig): TbEditorCompleter { + return new TbEditorCompleter(Object.keys(argumentsObj).reduce((acc, key) => { + acc[key] = argumentsObj[key].refEntityKey.type === ArgumentType.Rolling + ? CalculatedFieldRollingValueArgumentAutocomplete + : CalculatedFieldSingleValueArgumentAutocomplete; + return acc; + }, {})) + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 2f707c0a2f..f1e6905aab 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -99,6 +99,7 @@ [functionArgs]="functionArgs$ | async" [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" + [editorCompleter]="argumentsEditorCompleter$ | async" helpId="calculated-field/expression_fn" >