diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 61d9586095..396ecd3771 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -31,6 +31,7 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.DeviceStateManager; +import org.thingsboard.rule.engine.api.JobManager; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.MqttClientSettings; import org.thingsboard.rule.engine.api.NotificationCenter; @@ -96,6 +97,7 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleNodeStateService; +import org.thingsboard.server.dao.job.JobService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; @@ -552,6 +554,14 @@ public class ActorSystemContext { @Getter private CalculatedFieldQueueService calculatedFieldQueueService; + @Autowired + @Getter + private JobService jobService; + + @Autowired + @Getter + private JobManager jobManager; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private int maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index c07134105c..dff0cd4cf1 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -24,6 +24,7 @@ import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.DeviceStateManager; +import org.thingsboard.rule.engine.api.JobManager; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.MqttClientSettings; import org.thingsboard.rule.engine.api.NotificationCenter; @@ -93,6 +94,7 @@ import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.job.JobService; import org.thingsboard.server.dao.mobile.MobileAppBundleService; import org.thingsboard.server.dao.mobile.MobileAppService; import org.thingsboard.server.dao.nosql.CassandraStatementTask; @@ -889,6 +891,16 @@ public class DefaultTbContext implements TbContext { return mainCtx.getCalculatedFieldQueueService(); } + @Override + public JobService getJobService() { + return mainCtx.getJobService(); + } + + @Override + public JobManager getJobManager() { + return mainCtx.getJobManager(); + } + @Override public boolean isExternalNodeForceAck() { return mainCtx.isExternalNodeForceAck(); diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 93bf5b35fe..e5ea69dcb5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -385,7 +385,7 @@ public abstract class BaseController { public void handleControllerException(Exception e, HttpServletResponse response) { ThingsboardException thingsboardException = handleException(e); if (thingsboardException.getErrorCode() == ThingsboardErrorCode.GENERAL && thingsboardException.getCause() instanceof Exception - && StringUtils.equals(thingsboardException.getCause().getMessage(), thingsboardException.getMessage())) { + && StringUtils.equals(thingsboardException.getCause().getMessage(), thingsboardException.getMessage())) { e = (Exception) thingsboardException.getCause(); } else { e = thingsboardException; @@ -430,7 +430,7 @@ public abstract class BaseController { if (exception instanceof ThingsboardException) { return (ThingsboardException) exception; } else if (exception instanceof IllegalArgumentException || exception instanceof IncorrectParameterException - || exception instanceof DataValidationException || cause instanceof IncorrectParameterException) { + || exception instanceof DataValidationException || cause instanceof IncorrectParameterException) { return new ThingsboardException(exception.getMessage(), ThingsboardErrorCode.BAD_REQUEST_PARAMS); } else if (exception instanceof MessagingException) { return new ThingsboardException("Unable to send mail", ThingsboardErrorCode.GENERAL); @@ -595,88 +595,39 @@ public abstract class BaseController { } } - protected void checkEntityId(EntityId entityId, Operation operation) throws ThingsboardException { + protected HasId checkEntityId(EntityId entityId, Operation operation) throws ThingsboardException { try { if (entityId == null) { throw new ThingsboardException("Parameter entityId can't be empty!", ThingsboardErrorCode.BAD_REQUEST_PARAMS); } validateId(entityId.getId(), id -> "Incorrect entityId " + id); - switch (entityId.getEntityType()) { - case ALARM: - checkAlarmId(new AlarmId(entityId.getId()), operation); - return; - case DEVICE: - checkDeviceId(new DeviceId(entityId.getId()), operation); - return; - case DEVICE_PROFILE: - checkDeviceProfileId(new DeviceProfileId(entityId.getId()), operation); - return; - case CUSTOMER: - checkCustomerId(new CustomerId(entityId.getId()), operation); - return; - case TENANT: - checkTenantId(TenantId.fromUUID(entityId.getId()), operation); - return; - case TENANT_PROFILE: - checkTenantProfileId(new TenantProfileId(entityId.getId()), operation); - return; - case RULE_CHAIN: - checkRuleChain(new RuleChainId(entityId.getId()), operation); - return; - case RULE_NODE: - checkRuleNode(new RuleNodeId(entityId.getId()), operation); - return; - case ASSET: - checkAssetId(new AssetId(entityId.getId()), operation); - return; - case ASSET_PROFILE: - checkAssetProfileId(new AssetProfileId(entityId.getId()), operation); - return; - case DASHBOARD: - checkDashboardId(new DashboardId(entityId.getId()), operation); - return; - case USER: - checkUserId(new UserId(entityId.getId()), operation); - return; - case ENTITY_VIEW: - checkEntityViewId(new EntityViewId(entityId.getId()), operation); - return; - case EDGE: - checkEdgeId(new EdgeId(entityId.getId()), operation); - return; - case WIDGETS_BUNDLE: - checkWidgetsBundleId(new WidgetsBundleId(entityId.getId()), operation); - return; - case WIDGET_TYPE: - checkWidgetTypeId(new WidgetTypeId(entityId.getId()), operation); - return; - case TB_RESOURCE: - checkResourceInfoId(new TbResourceId(entityId.getId()), operation); - return; - case OTA_PACKAGE: - checkOtaPackageId(new OtaPackageId(entityId.getId()), operation); - return; - case QUEUE: - checkQueueId(new QueueId(entityId.getId()), operation); - return; - case OAUTH2_CLIENT: - checkOauth2ClientId(new OAuth2ClientId(entityId.getId()), operation); - return; - case DOMAIN: - checkDomainId(new DomainId(entityId.getId()), operation); - return; - case MOBILE_APP: - checkMobileAppId(new MobileAppId(entityId.getId()), operation); - return; - case MOBILE_APP_BUNDLE: - checkMobileAppBundleId(new MobileAppBundleId(entityId.getId()), operation); - return; - case CALCULATED_FIELD: - checkCalculatedFieldId(new CalculatedFieldId(entityId.getId()), operation); - return; - default: - checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); - } + return switch (entityId.getEntityType()) { + case ALARM -> checkAlarmId(new AlarmId(entityId.getId()), operation); + case DEVICE -> checkDeviceId(new DeviceId(entityId.getId()), operation); + case DEVICE_PROFILE -> checkDeviceProfileId(new DeviceProfileId(entityId.getId()), operation); + case CUSTOMER -> checkCustomerId(new CustomerId(entityId.getId()), operation); + case TENANT -> checkTenantId(TenantId.fromUUID(entityId.getId()), operation); + case TENANT_PROFILE -> checkTenantProfileId(new TenantProfileId(entityId.getId()), operation); + case RULE_CHAIN -> checkRuleChain(new RuleChainId(entityId.getId()), operation); + case RULE_NODE -> checkRuleNode(new RuleNodeId(entityId.getId()), operation); + case ASSET -> checkAssetId(new AssetId(entityId.getId()), operation); + case ASSET_PROFILE -> checkAssetProfileId(new AssetProfileId(entityId.getId()), operation); + case DASHBOARD -> checkDashboardId(new DashboardId(entityId.getId()), operation); + case USER -> checkUserId(new UserId(entityId.getId()), operation); + case ENTITY_VIEW -> checkEntityViewId(new EntityViewId(entityId.getId()), operation); + case EDGE -> checkEdgeId(new EdgeId(entityId.getId()), operation); + case WIDGETS_BUNDLE -> checkWidgetsBundleId(new WidgetsBundleId(entityId.getId()), operation); + case WIDGET_TYPE -> checkWidgetTypeId(new WidgetTypeId(entityId.getId()), operation); + case TB_RESOURCE -> checkResourceInfoId(new TbResourceId(entityId.getId()), operation); + case OTA_PACKAGE -> checkOtaPackageId(new OtaPackageId(entityId.getId()), operation); + case QUEUE -> checkQueueId(new QueueId(entityId.getId()), operation); + case OAUTH2_CLIENT -> checkOauth2ClientId(new OAuth2ClientId(entityId.getId()), operation); + case DOMAIN -> checkDomainId(new DomainId(entityId.getId()), operation); + case MOBILE_APP -> checkMobileAppId(new MobileAppId(entityId.getId()), operation); + case MOBILE_APP_BUNDLE -> checkMobileAppBundleId(new MobileAppBundleId(entityId.getId()), operation); + case CALCULATED_FIELD -> checkCalculatedFieldId(new CalculatedFieldId(entityId.getId()), operation); + default -> (HasId) checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); + }; } catch (Exception e) { throw handleException(e, false); } @@ -957,12 +908,13 @@ public abstract class BaseController { } } - private void checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { + private CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { validateId(calculatedFieldId, "Invalid entity id"); SecurityUser user = getCurrentUser(); CalculatedField cf = calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); checkNotNull(cf, calculatedFieldId.getEntityType().getNormalName() + " with id [" + calculatedFieldId + "] is not found"); checkEntityId(cf.getEntityId(), operation); + return cf; } protected HomeDashboardInfo getHomeDashboardInfo(SecurityUser securityUser, JsonNode additionalInfo) { diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index de7f25023f..3257b31ca4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -98,7 +98,8 @@ public class CalculatedFieldController extends BaseController { public static final int TIMEOUT = 20; - private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" + private static final String TEST_SCRIPT_EXPRESSION = + "Execute the Script expression and return the result. The format of request: \n\n" + MARKDOWN_CODE_BLOCK_START + "{\n" + " \"expression\": \"var temp = 0; foreach(element: temperature.values) {temp += element.value;} var avgTemperature = temp / temperature.values.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity.value; return {\\\"adjustedTemperature\\\": adjustedTemperature};\",\n" + diff --git a/application/src/main/java/org/thingsboard/server/controller/JobController.java b/application/src/main/java/org/thingsboard/server/controller/JobController.java new file mode 100644 index 0000000000..3f3ca5e64f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/JobController.java @@ -0,0 +1,111 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import io.swagger.v3.oas.annotations.Parameter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.rule.engine.api.JobManager; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobFilter; +import org.thingsboard.server.common.data.job.JobStatus; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.job.JobService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.List; +import java.util.UUID; + +import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +@RequiredArgsConstructor +@Slf4j +public class JobController extends BaseController { + + private final JobService jobService; + private final JobManager jobManager; + + @GetMapping("/job/{id}") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + public Job getJobById(@PathVariable UUID id) throws ThingsboardException { + return jobService.findJobById(getTenantId(), new JobId(id)); + } + + @GetMapping("/jobs") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + public PageData getJobs(@Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @Parameter(description = "Case-insensitive 'substring' filter based on job's description") + @RequestParam(required = false) String textSearch, + @Parameter(description = SORT_PROPERTY_DESCRIPTION) + @RequestParam(required = false) String sortProperty, + @Parameter(description = SORT_ORDER_DESCRIPTION) + @RequestParam(required = false) String sortOrder, + @RequestParam(required = false) List types, + @RequestParam(required = false) List statuses, + @RequestParam(required = false) List entities, + @RequestParam(required = false) Long startTime, + @RequestParam(required = false) Long endTime) throws ThingsboardException { + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + JobFilter filter = JobFilter.builder() + .types(types) + .statuses(statuses) + .entities(entities) + .startTime(startTime) + .endTime(endTime) + .build(); + return jobService.findJobsByFilter(getTenantId(), filter, pageLink); + } + + @PostMapping("/job/{id}/cancel") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + public void cancelJob(@PathVariable UUID id) throws ThingsboardException { + jobManager.cancelJob(getTenantId(), new JobId(id)); + } + + @PostMapping("/job/{id}/reprocess") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + public void reprocessJob(@PathVariable UUID id) throws ThingsboardException { + jobManager.reprocessJob(getTenantId(), new JobId(id)); + } + + @DeleteMapping("/job/{id}") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + public void deleteJob(@PathVariable UUID id) throws ThingsboardException { + jobService.deleteJob(getTenantId(), new JobId(id)); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 6ca767dc01..bf9713f58e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -64,11 +64,9 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AggregationParams; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DataType; @@ -78,7 +76,6 @@ import org.thingsboard.server.common.data.kv.IntervalType; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; @@ -92,6 +89,7 @@ import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.telemetry.AttributeData; +import org.thingsboard.server.service.telemetry.TbTelemetryService; import org.thingsboard.server.service.telemetry.TsData; import java.util.ArrayList; @@ -155,6 +153,9 @@ public class TelemetryController extends BaseController { @Autowired private AccessValidator accessValidator; + @Autowired + private TbTelemetryService tbTelemetryService; + @Value("${transport.json.max_string_value_length:0}") private int maxStringValueLength; @@ -313,30 +314,21 @@ public class TelemetryController extends BaseController { @Parameter(description = "A string value representing the timezone that will be used to calculate exact timestamps for 'WEEK', 'WEEK_ISO', 'MONTH' and 'QUARTER' interval types.") @RequestParam(name = "timeZone", required = false) String timeZone, @Parameter(description = "An integer value that represents a max number of time series data points to fetch." + - " This parameter is used only in the case if 'agg' parameter is set to 'NONE'.", schema = @Schema(defaultValue = "100")) + " This parameter is used only in the case if 'agg' parameter is set to 'NONE'.", schema = @Schema(defaultValue = "100")) @RequestParam(name = "limit", defaultValue = "100") Integer limit, @Parameter(description = "A string value representing the aggregation function. " + - "If the interval is not specified, 'agg' parameter will use 'NONE' value.", + "If the interval is not specified, 'agg' parameter will use 'NONE' value.", schema = @Schema(allowableValues = {"MIN", "MAX", "AVG", "SUM", "COUNT", "NONE"})) @RequestParam(name = "agg", defaultValue = "NONE") String aggStr, @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) @RequestParam(name = "orderBy", defaultValue = "DESC") String orderBy, @Parameter(description = STRICT_DATA_TYPES_DESCRIPTION) @RequestParam(name = "useStrictDataTypes", required = false, defaultValue = "false") Boolean useStrictDataTypes) throws ThingsboardException { - return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr, - (result, tenantId, entityId) -> { - AggregationParams params; - Aggregation agg = Aggregation.valueOf(aggStr); - if (Aggregation.NONE.equals(agg)) { - params = AggregationParams.none(); - } else if (intervalType == null || IntervalType.MILLISECONDS.equals(intervalType)) { - params = interval == 0L ? AggregationParams.none() : AggregationParams.milliseconds(agg, interval); - } else { - params = AggregationParams.calendar(agg, intervalType, timeZone); - } - List queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, params, limit, orderBy)).collect(Collectors.toList()); - Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor()); - }); + DeferredResult response = new DeferredResult<>(); + Futures.addCallback(tbTelemetryService.getTimeseries(EntityIdFactory.getByTypeAndId(entityType, entityIdStr), toKeysList(keys), startTs, endTs, + intervalType, interval, timeZone, limit, Aggregation.valueOf(aggStr), orderBy, useStrictDataTypes, getCurrentUser()), + getTsKvListCallback(response, useStrictDataTypes), MoreExecutors.directExecutor()); + return response; } @ApiOperation(value = "Save device attributes (saveDeviceAttributes)", @@ -345,12 +337,12 @@ public class TelemetryController extends BaseController { + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = SAVE_ATTIRIBUTES_STATUS_OK + - "Platform creates an audit log event about device attributes updates with action type 'ATTRIBUTES_UPDATED', " + - "and also sends event msg to the rule engine with msg type 'ATTRIBUTES_UPDATED'."), + "Platform creates an audit log event about device attributes updates with action type 'ATTRIBUTES_UPDATED', " + + "and also sends event msg to the rule engine with msg type 'ATTRIBUTES_UPDATED'."), @ApiResponse(responseCode = "400", description = SAVE_ATTIRIBUTES_STATUS_BAD_REQUEST), @ApiResponse(responseCode = "401", description = "User is not authorized to save device attributes for selected device. Most likely, User belongs to different Customer or Tenant."), @ApiResponse(responseCode = "500", description = "The exception was thrown during processing the request. " + - "Platform creates an audit log event about device attributes updates with action type 'ATTRIBUTES_UPDATED' that includes an error stacktrace."), + "Platform creates an audit log event about device attributes updates with action type 'ATTRIBUTES_UPDATED' that includes an error stacktrace."), }) @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/{deviceId}/{scope}", method = RequestMethod.POST) @@ -468,11 +460,11 @@ public class TelemetryController extends BaseController { TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "Time series for the selected keys in the request was removed. " + - "Platform creates an audit log event about entity time series removal with action type 'TIMESERIES_DELETED'."), + "Platform creates an audit log event about entity time series removal with action type 'TIMESERIES_DELETED'."), @ApiResponse(responseCode = "400", description = "Platform returns a bad request in case if keys list is empty or start and end timestamp values is empty when deleteAllDataForKeys is set to false."), @ApiResponse(responseCode = "401", description = "User is not authorized to delete entity time series for selected entity. Most likely, User belongs to different Customer or Tenant."), @ApiResponse(responseCode = "500", description = "The exception was thrown during processing the request. " + - "Platform creates an audit log event about entity time series removal with action type 'TIMESERIES_DELETED' that includes an error stacktrace."), + "Platform creates an audit log event about entity time series removal with action type 'TIMESERIES_DELETED' that includes an error stacktrace."), }) @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/{entityType}/{entityId}/timeseries/delete", method = RequestMethod.DELETE) @@ -549,11 +541,11 @@ public class TelemetryController extends BaseController { "Referencing a non-existing Device Id will cause an error" + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "Device attributes was removed for the selected keys in the request. " + - "Platform creates an audit log event about device attributes removal with action type 'ATTRIBUTES_DELETED'."), + "Platform creates an audit log event about device attributes removal with action type 'ATTRIBUTES_DELETED'."), @ApiResponse(responseCode = "400", description = "Platform returns a bad request in case if keys or scope are not specified."), @ApiResponse(responseCode = "401", description = "User is not authorized to delete device attributes for selected entity. Most likely, User belongs to different Customer or Tenant."), @ApiResponse(responseCode = "500", description = "The exception was thrown during processing the request. " + - "Platform creates an audit log event about device attributes removal with action type 'ATTRIBUTES_DELETED' that includes an error stacktrace."), + "Platform creates an audit log event about device attributes removal with action type 'ATTRIBUTES_DELETED' that includes an error stacktrace."), }) @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/{deviceId}/{scope}", method = RequestMethod.DELETE) @@ -571,11 +563,11 @@ public class TelemetryController extends BaseController { INVALID_ENTITY_ID_OR_ENTITY_TYPE_DESCRIPTION + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "Entity attributes was removed for the selected keys in the request. " + - "Platform creates an audit log event about entity attributes removal with action type 'ATTRIBUTES_DELETED'."), + "Platform creates an audit log event about entity attributes removal with action type 'ATTRIBUTES_DELETED'."), @ApiResponse(responseCode = "400", description = "Platform returns a bad request in case if keys or scope are not specified."), @ApiResponse(responseCode = "401", description = "User is not authorized to delete entity attributes for selected entity. Most likely, User belongs to different Customer or Tenant."), @ApiResponse(responseCode = "500", description = "The exception was thrown during processing the request. " + - "Platform creates an audit log event about entity attributes removal with action type 'ATTRIBUTES_DELETED' that includes an error stacktrace."), + "Platform creates an audit log event about entity attributes removal with action type 'ATTRIBUTES_DELETED' that includes an error stacktrace."), }) @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/{entityType}/{entityId}/{scope}", method = RequestMethod.DELETE) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index 871e1fa5e7..88f344d048 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -95,7 +95,7 @@ public abstract class EdqsSyncService { syncLatestTimeseries(); counters.clear(); - log.info("Finishing synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs)); + log.info("Finished synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs)); } private void process(TenantId tenantId, ObjectType type, EdqsObject object) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 648e89adc9..03ab77ac09 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.Job; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.notification.NotificationRequest; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -58,6 +59,7 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.rule.engine.api.JobManager; import java.util.Set; @@ -69,6 +71,7 @@ public class EntityStateSourcingListener { private final TenantService tenantService; private final TbClusterService tbClusterService; private final EdgeSynchronizationManager edgeSynchronizationManager; + private final JobManager jobManager; @PostConstruct public void init() { @@ -134,6 +137,9 @@ public class EntityStateSourcingListener { case CALCULATED_FIELD -> { onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); } + case JOB -> { + onJobUpdate((Job) event.getEntity()); + } default -> { } } @@ -212,8 +218,8 @@ public class EntityStateSourcingListener { public void handleEvent(ActionEntityEvent event) { log.trace("[{}] ActionEntityEvent called: {}", event.getTenantId(), event); if (ActionType.CREDENTIALS_UPDATED.equals(event.getActionType()) && - EntityType.DEVICE.equals(event.getEntityId().getEntityType()) - && event.getEntity() instanceof DeviceCredentials) { + EntityType.DEVICE.equals(event.getEntityId().getEntityType()) + && event.getEntity() instanceof DeviceCredentials) { tbClusterService.pushMsgToCore(new DeviceCredentialsUpdateNotificationMsg(event.getTenantId(), (DeviceId) event.getEntityId(), (DeviceCredentials) event.getEntity()), null); } else if (ActionType.ASSIGNED_TO_TENANT.equals(event.getActionType()) && event.getEntity() instanceof Device device) { @@ -295,6 +301,28 @@ public class EntityStateSourcingListener { tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, TbQueueCallback.EMPTY); } + private void onJobUpdate(Job job) { + jobManager.onJobUpdate(job); + + ComponentLifecycleEvent event; + if (job.getResult().getCancellationTs() > 0) { + event = ComponentLifecycleEvent.STOPPED; + } else if (job.getResult().getGeneralError() != null) { + event = ComponentLifecycleEvent.FAILED; + } else { + return; + } + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(job.getTenantId()) + .entityId(job.getId()) + .event(event) + .info(JacksonUtil.newObjectNode() + .put("tasksKey", job.getConfiguration().getTasksKey())) + .build(); + // task processors will add this job to the list of discarded + tbClusterService.broadcast(msg); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java index 727e27c971..f2d3fad357 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java @@ -165,6 +165,7 @@ public class HousekeeperService { private void stop() { consumer.stop(); consumerExecutor.shutdownNow(); + taskExecutor.shutdownNow(); log.info("Stopped Housekeeper service"); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/JobsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/JobsDeletionTaskProcessor.java new file mode 100644 index 0000000000..f0f829770a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/JobsDeletionTaskProcessor.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.dao.job.JobService; + +@Component +@RequiredArgsConstructor +@Slf4j +public class JobsDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final JobService jobService; + + @Override + public void process(HousekeeperTask task) throws Exception { + int deletedCount = jobService.deleteJobsByEntityId(task.getTenantId(), task.getEntityId()); + log.debug("[{}][{}][{}] Deleted {} jobs", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_JOBS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/job/DefaultJobManager.java b/application/src/main/java/org/thingsboard/server/service/job/DefaultJobManager.java new file mode 100644 index 0000000000..ed8e613f38 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/job/DefaultJobManager.java @@ -0,0 +1,207 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.job; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.rule.engine.api.JobManager; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobResult; +import org.thingsboard.server.common.data.job.JobStatus; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.job.task.Task; +import org.thingsboard.server.common.data.job.task.TaskResult; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.job.JobService; +import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.settings.TasksQueueConfig; +import org.thingsboard.server.queue.task.JobStatsService; +import org.thingsboard.server.queue.task.TaskProducerQueueFactory; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class DefaultJobManager implements JobManager { + + private final JobService jobService; + private final JobStatsService jobStatsService; + private final PartitionService partitionService; + private final TasksQueueConfig queueConfig; + private final Map jobProcessors; + private final Map>> taskProducers; + private final ExecutorService executor; + + public DefaultJobManager(JobService jobService, JobStatsService jobStatsService, PartitionService partitionService, + TaskProducerQueueFactory queueFactory, TasksQueueConfig queueConfig, + List jobProcessors) { + this.jobService = jobService; + this.jobStatsService = jobStatsService; + this.partitionService = partitionService; + this.queueConfig = queueConfig; + this.jobProcessors = jobProcessors.stream().collect(Collectors.toMap(JobProcessor::getType, Function.identity())); + this.taskProducers = Arrays.stream(JobType.values()).collect(Collectors.toMap(Function.identity(), queueFactory::createTaskProducer)); + this.executor = ThingsBoardExecutors.newWorkStealingPool(Math.max(4, Runtime.getRuntime().availableProcessors()), getClass()); + } + + @Override + public ListenableFuture submitJob(Job job) { + log.debug("Submitting job: {}", job); + return Futures.submit(() -> jobService.saveJob(job.getTenantId(), job), executor); + } + + @Override + public void onJobUpdate(Job job) { + JobStatus status = job.getStatus(); + switch (status) { + case PENDING -> { + executor.execute(() -> { + try { + processJob(job); + } catch (Throwable e) { + log.error("Failed to process job update: {}", job, e); + } + }); + } + case COMPLETED, FAILED -> { + executor.execute(() -> { + try { + getJobProcessor(job.getType()).onJobFinished(job); + } catch (Throwable e) { + log.error("Failed to process job update: {}", job, e); + } + }); + } + } + } + + private void processJob(Job job) { + TenantId tenantId = job.getTenantId(); + JobId jobId = job.getId(); + try { + JobProcessor processor = getJobProcessor(job.getType()); + List toReprocess = job.getConfiguration().getToReprocess(); + if (toReprocess == null) { + int tasksCount = processor.process(job, this::submitTask); + log.info("[{}][{}][{}] Submitted {} tasks", tenantId, jobId, job.getType(), tasksCount); + jobStatsService.reportAllTasksSubmitted(tenantId, jobId, tasksCount); + } else { + processor.reprocess(job, toReprocess, this::submitTask); + log.info("[{}][{}][{}] Submitted {} tasks for reprocessing", tenantId, jobId, job.getType(), toReprocess.size()); + } + } catch (Throwable e) { + log.error("[{}][{}][{}] Failed to submit tasks", tenantId, jobId, job.getType(), e); + jobService.markAsFailed(tenantId, jobId, e.getMessage()); + } + } + + @Override + public void cancelJob(TenantId tenantId, JobId jobId) { + log.info("[{}][{}] Cancelling job", tenantId, jobId); + jobService.cancelJob(tenantId, jobId); + } + + @Override + public void reprocessJob(TenantId tenantId, JobId jobId) { + log.info("[{}][{}] Reprocessing job", tenantId, jobId); + Job job = jobService.findJobById(tenantId, jobId); + if (job.getStatus() != JobStatus.FAILED) { + throw new IllegalArgumentException("Job is not failed"); + } + + JobResult result = job.getResult(); + if (result.getGeneralError() != null) { + job.presetResult(); + } else { + List taskFailures = result.getResults().stream() + .filter(taskResult -> !taskResult.isSuccess() && !taskResult.isDiscarded()) + .toList(); + if (result.getFailedCount() > taskFailures.size()) { + throw new IllegalArgumentException("Reprocessing not allowed since there are too many failures (more than " + taskFailures.size() + ")"); + } + result.setFailedCount(0); + result.setResults(result.getResults().stream() + .filter(TaskResult::isSuccess) + .toList()); + job.getConfiguration().setToReprocess(taskFailures); + } + job.getConfiguration().setTasksKey(UUID.randomUUID().toString()); + jobService.saveJob(tenantId, job); + } + + private void submitTask(Task task) { + if (ObjectUtils.anyNull(task.getTenantId(), task.getJobId(), task.getKey())) { + throw new IllegalArgumentException("Task " + task + " missing required fields"); + } + + log.debug("[{}][{}] Submitting task: {}", task.getTenantId(), task.getJobId(), task); + TaskProto taskProto = TaskProto.newBuilder() + .setValue(JacksonUtil.toString(task)) + .build(); + + TbQueueProducer> producer = taskProducers.get(task.getJobType()); + EntityId entityId = null; + if (queueConfig.getPartitioningStrategy().equals("entity")) { + entityId = task.getEntityId(); + } + if (entityId == null) { + entityId = task.getTenantId(); + } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TASK_PROCESSOR, task.getJobType().name(), task.getTenantId(), entityId); + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), taskProto), new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.trace("Submitted task to {}: {}", tpi, taskProto); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to submit task: {}", task, t); + } + }); + } + + private JobProcessor getJobProcessor(JobType jobType) { + return jobProcessors.get(jobType); + } + + @PreDestroy + private void destroy() { + executor.shutdownNow(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/job/DummyJobProcessor.java b/application/src/main/java/org/thingsboard/server/service/job/DummyJobProcessor.java new file mode 100644 index 0000000000..373bc4afee --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/job/DummyJobProcessor.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.job; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.job.DummyJobConfiguration; +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.job.task.DummyTask; +import org.thingsboard.server.common.data.job.task.DummyTaskResult; +import org.thingsboard.server.common.data.job.task.DummyTaskResult.DummyTaskFailure; +import org.thingsboard.server.common.data.job.task.Task; +import org.thingsboard.server.common.data.job.task.TaskResult; + +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; + +@Component +@RequiredArgsConstructor +public class DummyJobProcessor implements JobProcessor { + + @Override + public int process(Job job, Consumer> taskConsumer) throws Exception { + DummyJobConfiguration configuration = job.getConfiguration(); + if (configuration.getGeneralError() != null) { + for (int number = 1; number <= configuration.getSubmittedTasksBeforeGeneralError(); number++) { + taskConsumer.accept(createTask(job, configuration, number, null, false)); + } + Thread.sleep(configuration.getTaskProcessingTimeMs() * (configuration.getSubmittedTasksBeforeGeneralError() / 2)); // sleeping so that some tasks are processed + throw new RuntimeException(configuration.getGeneralError()); + } + + int taskNumber = 1; + for (int i = 0; i < configuration.getSuccessfulTasksCount(); i++) { + taskConsumer.accept(createTask(job, configuration, taskNumber, null, false)); + taskNumber++; + } + if (configuration.getErrors() != null) { + for (int i = 0; i < configuration.getFailedTasksCount(); i++) { + taskConsumer.accept(createTask(job, configuration, taskNumber, configuration.getErrors(), false)); + taskNumber++; + } + for (int i = 0; i < configuration.getPermanentlyFailedTasksCount(); i++) { + taskConsumer.accept(createTask(job, configuration, taskNumber, configuration.getErrors(), true)); + taskNumber++; + } + } + return configuration.getSuccessfulTasksCount() + configuration.getFailedTasksCount() + configuration.getPermanentlyFailedTasksCount(); + } + + @Override + public void reprocess(Job job, List taskFailures, Consumer> taskConsumer) throws Exception { + for (TaskResult taskFailure : taskFailures) { + DummyTaskFailure failure = ((DummyTaskResult) taskFailure).getFailure(); + taskConsumer.accept(createTask(job, job.getConfiguration(), failure.getNumber(), failure.isFailAlways() ? + List.of(failure.getError()) : Collections.emptyList(), failure.isFailAlways())); + } + } + + private DummyTask createTask(Job job, DummyJobConfiguration configuration, int number, List errors, boolean failAlways) { + return DummyTask.builder() + .tenantId(job.getTenantId()) + .jobId(job.getId()) + .key(configuration.getTasksKey()) + .retries(configuration.getRetries()) + .number(number) + .processingTimeMs(configuration.getTaskProcessingTimeMs()) + .errors(errors) + .failAlways(failAlways) + .build(); + } + + @Override + public JobType getType() { + return JobType.DUMMY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/job/JobProcessor.java b/application/src/main/java/org/thingsboard/server/service/job/JobProcessor.java new file mode 100644 index 0000000000..e33878d008 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/job/JobProcessor.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.job; + +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.job.task.Task; +import org.thingsboard.server.common.data.job.task.TaskResult; + +import java.util.List; +import java.util.function.Consumer; + +public interface JobProcessor { + + int process(Job job, Consumer> taskConsumer) throws Exception; + + void reprocess(Job job, List taskFailures, Consumer> taskConsumer) throws Exception; + + default void onJobFinished(Job job) {} + + JobType getType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/job/JobStatsProcessor.java b/application/src/main/java/org/thingsboard/server/service/job/JobStatsProcessor.java new file mode 100644 index 0000000000..c5b2577819 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/job/JobStatsProcessor.java @@ -0,0 +1,115 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.job; + +import jakarta.annotation.PreDestroy; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.JobStats; +import org.thingsboard.server.common.data.job.task.TaskResult; +import org.thingsboard.server.dao.job.JobService; +import org.thingsboard.server.gen.transport.TransportProtos.JobStatsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.settings.TasksQueueConfig; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@TbCoreComponent +@Component +@Slf4j +public class JobStatsProcessor { + + private final JobService jobService; + private final TasksQueueConfig queueConfig; + private final QueueConsumerManager> jobStatsConsumer; + private final ExecutorService consumerExecutor; + + public JobStatsProcessor(JobService jobService, + TasksQueueConfig queueConfig, + TbCoreQueueFactory queueFactory) { + this.jobService = jobService; + this.queueConfig = queueConfig; + this.consumerExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("job-stats-consumer")); + this.jobStatsConsumer = QueueConsumerManager.>builder() + .name("job-stats") + .msgPackProcessor(this::processStats) + .pollInterval(queueConfig.getStatsPollInterval()) + .consumerCreator(queueFactory::createJobStatsConsumer) + .consumerExecutor(consumerExecutor) + .build(); + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + jobStatsConsumer.subscribe(); + jobStatsConsumer.launch(); + } + + @SneakyThrows + private void processStats(List> msgs, TbQueueConsumer> consumer) { + Map stats = new HashMap<>(); + + for (TbProtoQueueMsg msg : msgs) { + JobStatsMsg statsMsg = msg.getValue(); + TenantId tenantId = TenantId.fromUUID(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB())); + JobId jobId = new JobId(new UUID(statsMsg.getJobIdMSB(), statsMsg.getJobIdLSB())); + JobStats jobStats = stats.computeIfAbsent(jobId, __ -> new JobStats(tenantId, jobId)); + + if (statsMsg.hasTaskResult()) { + TaskResult taskResult = JacksonUtil.fromString(statsMsg.getTaskResult().getValue(), TaskResult.class); + jobStats.getTaskResults().add(taskResult); + } + if (statsMsg.hasTotalTasksCount()) { + jobStats.setTotalTasksCount(statsMsg.getTotalTasksCount()); + } + } + + stats.forEach((jobId, jobStats) -> { + TenantId tenantId = jobStats.getTenantId(); + try { + log.debug("[{}][{}] Processing job stats: {}", tenantId, jobId, stats); + jobService.processStats(tenantId, jobId, jobStats); + } catch (Exception e) { + log.error("[{}][{}] Failed to process job stats: {}", tenantId, jobId, jobStats, e); + } + }); + consumer.commit(); + + Thread.sleep(queueConfig.getStatsProcessingInterval()); + } + + @PreDestroy + private void destroy() { + jobStatsConsumer.stop(); + consumerExecutor.shutdownNow(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/job/task/DummyTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/job/task/DummyTaskProcessor.java new file mode 100644 index 0000000000..5c88083146 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/job/task/DummyTaskProcessor.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.job.task; + +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.job.task.DummyTask; +import org.thingsboard.server.common.data.job.task.DummyTaskResult; +import org.thingsboard.server.queue.task.TaskProcessor; + +@RequiredArgsConstructor +public class DummyTaskProcessor extends TaskProcessor { + + @Override + public DummyTaskResult process(DummyTask task) throws Exception { + if (task.getProcessingTimeMs() > 0) { + Thread.sleep(task.getProcessingTimeMs()); + } + if (task.isFailAlways()) { + throw new RuntimeException(task.getErrors().get(0)); + } + if (task.getErrors() != null && task.getAttempt() <= task.getErrors().size()) { + String error = task.getErrors().get(task.getAttempt() - 1); + throw new RuntimeException(error); + } + return DummyTaskResult.success(task); + } + + @Override + public long getTaskProcessingTimeout() { + return 2000; + } + + @Override + public JobType getJobType() { + return JobType.DUMMY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 561dc7122a..a14ba10bfb 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -579,7 +579,8 @@ public class DefaultTbClusterService implements TbClusterService { } } - private void broadcast(ComponentLifecycleMsg msg) { + @Override + public void broadcast(ComponentLifecycleMsg msg) { ComponentLifecycleMsgProto componentLifecycleMsgProto = toProto(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); @@ -594,6 +595,7 @@ public class DefaultTbClusterService implements TbClusterService { || entityType.equals(EntityType.ENTITY_VIEW) || entityType.equals(EntityType.NOTIFICATION_RULE) || entityType.equals(EntityType.CALCULATED_FIELD) + || entityType.equals(EntityType.JOB) ) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTbTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTbTelemetryService.java new file mode 100644 index 0000000000..d55f44a027 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTbTelemetryService.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AggregationParams; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.IntervalType; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.service.security.AccessValidator; +import org.thingsboard.server.service.security.ValidationResult; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; + +import java.util.List; +import java.util.stream.Collectors; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultTbTelemetryService implements TbTelemetryService { + + private final TimeseriesService tsService; + private final AccessValidator accessValidator; + + @Override + public ListenableFuture> getTimeseries(EntityId entityId, List keys, Long startTs, Long endTs, IntervalType intervalType, + Long interval, String timeZone, Integer limit, Aggregation agg, String orderBy, + Boolean useStrictDataTypes, SecurityUser currentUser) { + SettableFuture> future = SettableFuture.create(); + accessValidator.validate(currentUser, Operation.READ_TELEMETRY, entityId, new FutureCallback<>() { + @Override + public void onSuccess(ValidationResult validationResult) { + try { + AggregationParams params; + if (Aggregation.NONE.equals(agg)) { + params = AggregationParams.none(); + } else if (intervalType == null || IntervalType.MILLISECONDS.equals(intervalType)) { + params = interval == 0L ? AggregationParams.none() : AggregationParams.milliseconds(agg, interval); + } else { + params = AggregationParams.calendar(agg, intervalType, timeZone); + } + List queries = keys.stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, params, limit, orderBy)).collect(Collectors.toList()); + Futures.addCallback(tsService.findAll(currentUser.getTenantId(), entityId, queries), new FutureCallback<>() { + @Override + public void onSuccess(List result) { + future.set(result); + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }, MoreExecutors.directExecutor()); + } catch (Throwable e) { + onFailure(e); + } + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }); + return future; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TbTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TbTelemetryService.java new file mode 100644 index 0000000000..9820e62592 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TbTelemetryService.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.IntervalType; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.List; + +public interface TbTelemetryService { + + ListenableFuture> getTimeseries(EntityId entityId, + List keys, + Long startTs, + Long endTs, + IntervalType intervalType, + Long interval, + String timeZone, + Integer limit, + Aggregation agg, + String orderBy, + Boolean useStrictDataTypes, + SecurityUser currentUser) throws ThingsboardException; + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ab64d18500..6d27b9bd7e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1631,6 +1631,11 @@ queue: - key: max.poll.records # Max poll records for edqs.state topic value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}" + tasks: + # Key-value properties for Kafka consumer for tasks topics + - key: max.poll.records + # Max poll records for tasks topics + value: "${TB_QUEUE_KAFKA_TASKS_MAX_POLL_RECORDS:1}" # If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties # for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc). # Each entry sets a single property for a specific topic. To define multiple properties for a topic, repeat the topic key. @@ -1676,6 +1681,8 @@ queue: edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for EDQS state topic (infinite retention, compaction) edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" + # Kafka properties for tasks topics + tasks: "${TB_QUEUE_KAFKA_TASKS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600;partitions:1;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1896,6 +1903,23 @@ queue: enabled: "${TB_QUEUE_EDGE_STATS_ENABLED:true}" # Statistics printing interval for Edge services print-interval-ms: "${TB_QUEUE_EDGE_STATS_PRINT_INTERVAL_MS:60000}" + tasks: + # Poll interval in milliseconds for tasks topics + poll_interval: "${TB_QUEUE_TASKS_POLL_INTERVAL_MS:500}" + # Partitions count for tasks queues + partitions: "${TB_QUEUE_TASKS_PARTITIONS:12}" + # Custom partitions count for tasks queues per type. Format: 'TYPE1:24;TYPE2:36', e.g. 'CF_REPROCESSING:24;TENANT_EXPORT:6' + partitions_per_type: "${TB_QUEUE_TASKS_PARTITIONS_PER_TYPE:}" + # Tasks partitioning strategy: 'tenant' or 'entity'. By default, using 'tenant' - tasks of a specific tenant are processed in the same partition. + # In a single-tenant environment, use 'entity' strategy to distribute the tasks among multiple partitions. + partitioning_strategy: "${TB_QUEUE_TASKS_PARTITIONING_STRATEGY:tenant}" + stats: + # Name for the tasks stats topic + topic: "${TB_QUEUE_TASKS_STATS_TOPIC:jobs.stats}" + # Poll interval in milliseconds for tasks stats topic + poll_interval: "${TB_QUEUE_TASKS_STATS_POLL_INTERVAL_MS:500}" + # Interval in milliseconds to process job stats + processing_interval: "${TB_QUEUE_TASKS_STATS_PROCESSING_INTERVAL_MS:1000}" # Event configuration parameters event: diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 33f2209eca..b93ff0f623 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -48,6 +48,7 @@ import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.mock.http.MockHttpInputMessage; import org.springframework.mock.http.MockHttpOutputMessage; import org.springframework.mock.web.MockMultipartFile; @@ -102,10 +103,13 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.JobId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobType; import org.thingsboard.server.common.data.notification.Notification; import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod; import org.thingsboard.server.common.data.notification.NotificationType; @@ -128,6 +132,7 @@ import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig; import org.thingsboard.server.common.data.oauth2.PlatformType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.security.Authority; @@ -164,6 +169,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -278,6 +284,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { @Autowired protected InMemoryStorage storage; + @Autowired + protected JdbcTemplate jdbcTemplate; + @MockBean protected CfRocksDb cfRocksDb; @@ -389,6 +398,8 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { tenantProfileService.deleteTenantProfiles(TenantId.SYS_TENANT_ID); + jdbcTemplate.execute("TRUNCATE TABLE notification"); + log.info("Executed web test teardown"); } @@ -1253,4 +1264,26 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { new PageLink(limit, 0), unreadOnly, deliveryMethod).getData(); } + protected Job findJobById(JobId jobId) throws Exception { + return doGet("/api/job/" + jobId, Job.class); + } + + protected List findJobs() throws Exception { + return doGetTypedWithPageLink("/api/jobs?", new TypeReference>() {}, new PageLink(100, 0, null, new SortOrder("createdTime", SortOrder.Direction.DESC))).getData(); + } + + protected List findJobs(List types, List entities) throws Exception { + return doGetTypedWithPageLink("/api/jobs?types=" + types.stream().map(Enum::name).collect(Collectors.joining(",")) + + "&entities=" + entities.stream().map(UUID::toString).collect(Collectors.joining(",")) + "&", + new TypeReference>() {}, new PageLink(100, 0, null, new SortOrder("createdTime", SortOrder.Direction.DESC))).getData(); + } + + protected void cancelJob(JobId jobId) throws Exception { + doPost("/api/job/" + jobId + "/cancel").andExpect(status().isOk()); + } + + protected void reprocessJob(JobId jobId) throws Exception { + doPost("/api/job/" + jobId + "/reprocess").andExpect(status().isOk()); + } + } diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index dace159774..e21457f65d 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -49,6 +49,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -419,10 +420,11 @@ public class HashPartitionServiceTest { } private HashPartitionService createPartitionService() { - HashPartitionService partitionService = new HashPartitionService(serviceInfoProvider, - routingInfoService, + HashPartitionService partitionService = new HashPartitionService( applicationEventPublisher, - queueRoutingInfoService, + serviceInfoProvider, + Optional.of(routingInfoService), + Optional.of(queueRoutingInfoService), topicService); ReflectionTestUtils.setField(partitionService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(partitionService, "corePartitions", 10); @@ -434,6 +436,7 @@ public class HashPartitionServiceTest { ReflectionTestUtils.setField(partitionService, "edgeTopic", "tb.edge"); ReflectionTestUtils.setField(partitionService, "edgePartitions", 10); ReflectionTestUtils.setField(partitionService, "edqsPartitions", 12); + ReflectionTestUtils.setField(partitionService, "defaultTasksPartitions", 12); partitionService.init(); partitionService.partitionsInit(); return partitionService; diff --git a/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java b/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java new file mode 100644 index 0000000000..0278e910d9 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java @@ -0,0 +1,478 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.job; + +import lombok.SneakyThrows; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.rule.engine.api.JobManager; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.DummyJobConfiguration; +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobFilter; +import org.thingsboard.server.common.data.job.JobResult; +import org.thingsboard.server.common.data.job.JobStatus; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.job.task.DummyTaskResult; +import org.thingsboard.server.common.data.job.task.DummyTaskResult.DummyTaskFailure; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.job.JobDao; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.queue.task.JobStatsService; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +@TestPropertySource(properties = { + "queue.tasks.stats.processing_interval=0" +}) +public class JobManagerTest extends AbstractControllerTest { + + @Autowired + private JobManager jobManager; + + @SpyBean + private TestTaskProcessor taskProcessor; + + @SpyBean + private JobStatsService jobStatsService; + + @Autowired + private JobDao jobDao; + + private TenantId tenantId; + private Device jobEntity; + + @Before + public void setUp() throws Exception { + loginTenantAdmin(); + tenantId = super.tenantId; + jobEntity = createDevice("Test", "Test"); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testSubmitJob_allTasksSuccessful() { + int tasksCount = 5; + JobId jobId = submitJob(DummyJobConfiguration.builder() + .successfulTasksCount(tasksCount) + .taskProcessingTimeMs(1000) + .build()).getId(); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.RUNNING); + assertThat(job.getResult().getSuccessfulCount()).isBetween(1, tasksCount - 1); + assertThat(job.getResult().getTotalCount()).isEqualTo(tasksCount); + }); + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.COMPLETED); + assertThat(job.getResult().getSuccessfulCount()).isEqualTo(tasksCount); + assertThat(job.getResult().getResults()).isEmpty(); + assertThat(job.getResult().getCompletedCount()).isEqualTo(tasksCount); + }); + } + + @Test + public void testSubmitJob_someTasksPermanentlyFailed() { + int successfulTasks = 3; + int failedTasks = 2; + JobId jobId = submitJob(DummyJobConfiguration.builder() + .successfulTasksCount(successfulTasks) + .failedTasksCount(failedTasks) + .errors(List.of("error1", "error2", "error3")) + .retries(2) + .taskProcessingTimeMs(100) + .build()).getId(); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.FAILED); + JobResult jobResult = job.getResult(); + assertThat(jobResult.getSuccessfulCount()).isEqualTo(successfulTasks); + assertThat(jobResult.getFailedCount()).isEqualTo(failedTasks); + assertThat(jobResult.getTotalCount()).isEqualTo(successfulTasks + failedTasks); + assertThat(getFailures(jobResult)).hasSize(2).allSatisfy(failure -> { + assertThat(failure.getError()).isEqualTo("error3"); // last error + }); + assertThat(jobResult.getCompletedCount()).isEqualTo(jobResult.getTotalCount()); + }); + } + + @Test + public void testSubmitJob_taskTimeout() { + JobId jobId = submitJob(DummyJobConfiguration.builder() + .successfulTasksCount(1) + .taskProcessingTimeMs(5000) // bigger than DummyTaskProcessor.getTaskProcessingTimeout() + .build()).getId(); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.FAILED); + JobResult jobResult = job.getResult(); + assertThat(jobResult.getFailedCount()).isEqualTo(1); + assertThat(((DummyTaskResult) jobResult.getResults().get(0)).getFailure().getError()).isEqualTo("Timeout after 2000 ms"); // last error + }); + } + + @Test + public void testCancelJob_whileRunning() throws Exception { + int tasksCount = 100; + JobId jobId = submitJob(DummyJobConfiguration.builder() + .successfulTasksCount(tasksCount) + .taskProcessingTimeMs(100) + .build()).getId(); + + Thread.sleep(500); + cancelJob(jobId); + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.CANCELLED); + assertThat(job.getResult().getSuccessfulCount()).isBetween(1, tasksCount - 1); + assertThat(job.getResult().getDiscardedCount()).isBetween(1, tasksCount - 1); + assertThat(job.getResult().getTotalCount()).isEqualTo(tasksCount); + assertThat(job.getResult().getCompletedCount()).isEqualTo(tasksCount); + }); + } + + @Test + public void testCancelJob_simulateTaskProcessorRestart() throws Exception { + int tasksCount = 10; + JobId jobId = submitJob(DummyJobConfiguration.builder() + .successfulTasksCount(tasksCount) + .taskProcessingTimeMs(500) + .build()).getId(); + + // simulate cancelled jobs are forgotten + AtomicInteger cancellationRenotifyAttempt = new AtomicInteger(0); + doAnswer(inv -> { + if (cancellationRenotifyAttempt.incrementAndGet() >= 5) { + inv.callRealMethod(); + } + return null; + }).when(taskProcessor).addToDiscarded(any()); // ignoring cancellation event, + cancelJob(jobId); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.CANCELLED); + assertThat(job.getResult().getSuccessfulCount()).isBetween(1, tasksCount - 1); + assertThat(job.getResult().getDiscardedCount()).isBetween(1, tasksCount - 1); + assertThat(job.getResult().getTotalCount()).isEqualTo(tasksCount); + assertThat(job.getResult().getCompletedCount()).isEqualTo(tasksCount); + }); + } + + @Test + public void whenTenantIsDeleted_thenCancelAllTheJobs() throws Exception { + loginSysAdmin(); + createDifferentTenant(); + + this.tenantId = this.differentTenantId; + submitJob(DummyJobConfiguration.builder() + .successfulTasksCount(1000) + .taskProcessingTimeMs(500) + .build()); + + Thread.sleep(2000); + deleteDifferentTenant(); + Mockito.reset(jobStatsService); + + Thread.sleep(3000); + verify(jobStatsService, never()).reportTaskResult(any(), any(), any()); + assertThat(jobDao.findByTenantIdAndFilter(tenantId, JobFilter.builder().build(), new PageLink(100)).getData()).isEmpty(); + } + + @Test + public void testSubmitMultipleJobs() throws Exception { + int tasksCount = 3; + int jobsCount = 3; + for (int i = 1; i <= jobsCount; i++) { + submitJob(DummyJobConfiguration.builder() + .successfulTasksCount(tasksCount) + .taskProcessingTimeMs(1000) + .build(), "test-job-" + i); + } + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + List jobs = findJobs(List.of(JobType.DUMMY), List.of(jobEntity.getUuidId())); + assertThat(jobs).hasSize(jobsCount); + Job firstJob = jobs.get(2); // ordered by createdTime descending + assertThat(firstJob.getStatus()).isEqualTo(JobStatus.RUNNING); + Job secondJob = jobs.get(1); + assertThat(secondJob.getStatus()).isEqualTo(JobStatus.QUEUED); + Job thirdJob = jobs.get(0); + assertThat(thirdJob.getStatus()).isEqualTo(JobStatus.QUEUED); + }); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + List jobs = findJobs(); + for (Job job : jobs) { + assertThat(job.getStatus()).isEqualTo(JobStatus.COMPLETED); + assertThat(job.getResult().getSuccessfulCount()).isEqualTo(tasksCount); + assertThat(job.getResult().getTotalCount()).isEqualTo(tasksCount); + assertThat(job.getEntityId()).isEqualTo(jobEntity.getId()); + assertThat(job.getEntityName()).isEqualTo(jobEntity.getName()); + } + }); + + doDelete("/api/device/" + jobEntity.getId()).andExpect(status().isOk()); + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + assertThat(findJobs(List.of(JobType.DUMMY), List.of(jobEntity.getUuidId()))).isEmpty(); + }); + } + + @Test + public void testCancelQueuedJob() throws Exception { + int tasksCount = 3; + int jobsCount = 3; + List jobIds = new ArrayList<>(); + for (int i = 1; i <= jobsCount; i++) { + Job job = submitJob(DummyJobConfiguration.builder() + .successfulTasksCount(tasksCount) + .taskProcessingTimeMs(1000) + .build(), "test-job-" + i); + jobIds.add(job.getId()); + } + + for (int i = 1; i < jobIds.size(); i++) { + cancelJob(jobIds.get(i)); + } + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + List jobs = findJobs(); + + Job firstJob = jobs.get(2); + assertThat(firstJob.getStatus()).isEqualTo(JobStatus.COMPLETED); + assertThat(firstJob.getResult().getSuccessfulCount()).isEqualTo(tasksCount); + assertThat(firstJob.getResult().getTotalCount()).isEqualTo(tasksCount); + + Job secondJob = jobs.get(1); + assertThat(secondJob.getStatus()).isEqualTo(JobStatus.CANCELLED); + assertThat(secondJob.getResult().getCompletedCount()).isZero(); + + Job thirdJob = jobs.get(0); + assertThat(thirdJob.getStatus()).isEqualTo(JobStatus.CANCELLED); + assertThat(thirdJob.getResult().getCompletedCount()).isZero(); + }); + } + + @Test + public void testSubmitJob_generalError() { + int submittedTasks = 100; + JobId jobId = submitJob(DummyJobConfiguration.builder() + .generalError("Some error while submitting tasks") + .submittedTasksBeforeGeneralError(submittedTasks) + .taskProcessingTimeMs(10) + .build()).getId(); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.FAILED); + assertThat(job.getResult().getSuccessfulCount()).isBetween(1, submittedTasks); + assertThat(job.getResult().getDiscardedCount()).isZero(); + assertThat(job.getResult().getTotalCount()).isNull(); + }); + } + + @Test + public void testSubmitJob_immediateGeneralError() { + JobId jobId = submitJob(DummyJobConfiguration.builder() + .generalError("Some error while submitting tasks") + .submittedTasksBeforeGeneralError(0) + .build()).getId(); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.FAILED); + assertThat(job.getResult().getSuccessfulCount()).isZero(); + assertThat(job.getResult().getDiscardedCount()).isZero(); + assertThat(job.getResult().getFailedCount()).isZero(); + assertThat(job.getResult().getTotalCount()).isNull(); + }); + } + + @Test + public void testReprocessJob_generalError() throws Exception { + int submittedTasks = 100; + JobId jobId = submitJob(DummyJobConfiguration.builder() + .generalError("Some error while submitting tasks") + .submittedTasksBeforeGeneralError(submittedTasks) + .taskProcessingTimeMs(10) + .build()).getId(); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.FAILED); + assertThat(job.getResult().getGeneralError()).isEqualTo("Some error while submitting tasks"); + }); + + Job savedJob = jobDao.findById(tenantId, jobId.getId()); + DummyJobConfiguration configuration = savedJob.getConfiguration(); + configuration.setGeneralError(null); + configuration.setSuccessfulTasksCount(submittedTasks); + jobDao.save(tenantId, savedJob); + + reprocessJob(jobId); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.COMPLETED); + assertThat(job.getResult().getGeneralError()).isNull(); + assertThat(job.getResult().getSuccessfulCount()).isEqualTo(submittedTasks); + assertThat(job.getResult().getTotalCount()).isEqualTo(submittedTasks); + assertThat(job.getResult().getFailedCount()).isZero(); + assertThat(job.getResult().getDiscardedCount()).isZero(); + }); + } + + @Test + public void testReprocessJob() throws Exception { + int successfulTasks = 3; + int failedTasks = 2; + int totalTasksCount = successfulTasks + failedTasks; + JobId jobId = submitJob(DummyJobConfiguration.builder() + .successfulTasksCount(successfulTasks) + .failedTasksCount(failedTasks) + .errors(List.of("error")) + .taskProcessingTimeMs(100) + .build()).getId(); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.FAILED); + JobResult jobResult = job.getResult(); + assertThat(jobResult.getSuccessfulCount()).isEqualTo(successfulTasks); + assertThat(jobResult.getFailedCount()).isEqualTo(failedTasks); + + List failures = getFailures(jobResult); + for (int i = 0, taskNumber = successfulTasks + 1; taskNumber <= totalTasksCount; i++, taskNumber++) { + DummyTaskFailure failure = failures.get(i); + assertThat(failure.getNumber()).isEqualTo(taskNumber); + assertThat(failure.getError()).isEqualTo("error"); + } + }); + + reprocessJob(jobId); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.COMPLETED); + assertThat(job.getResult().getSuccessfulCount()).isEqualTo(totalTasksCount); + assertThat(job.getResult().getFailedCount()).isZero(); + assertThat(job.getResult().getTotalCount()).isEqualTo(totalTasksCount); + assertThat(job.getResult().getResults()).isEmpty(); + assertThat(job.getConfiguration().getToReprocess()).isNullOrEmpty(); + }); + } + + @Test + public void testReprocessJob_somePermanentlyFailed() throws Exception { + int successfulTasks = 3; + int failedTasks = 2; + int permanentlyFailedTasks = 1; + int totalTasksCount = successfulTasks + failedTasks + permanentlyFailedTasks; + JobId jobId = submitJob(DummyJobConfiguration.builder() + .successfulTasksCount(successfulTasks) + .failedTasksCount(failedTasks) + .permanentlyFailedTasksCount(permanentlyFailedTasks) + .errors(List.of("error")) + .taskProcessingTimeMs(100) + .build()).getId(); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.FAILED); + JobResult jobResult = job.getResult(); + assertThat(jobResult.getSuccessfulCount()).isEqualTo(successfulTasks); + assertThat(jobResult.getFailedCount()).isEqualTo(failedTasks + permanentlyFailedTasks); + assertThat(jobResult.getTotalCount()).isEqualTo(totalTasksCount); + + List failures = getFailures(jobResult); + for (int i = 0, taskNumber = successfulTasks + 1; taskNumber <= totalTasksCount; i++, taskNumber++) { + DummyTaskFailure failure = failures.get(i); + assertThat(failure.getNumber()).isEqualTo(taskNumber); + assertThat(failure.getError()).isEqualTo("error"); + } + }); + + reprocessJob(jobId); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Job job = findJobById(jobId); + assertThat(job.getStatus()).isEqualTo(JobStatus.FAILED); + JobResult jobResult = job.getResult(); + assertThat(jobResult.getSuccessfulCount()).isEqualTo(successfulTasks + failedTasks); + assertThat(jobResult.getFailedCount()).isEqualTo(permanentlyFailedTasks); + assertThat(jobResult.getTotalCount()).isEqualTo(totalTasksCount); + + List failures = getFailures(jobResult); + for (int i = 0, taskNumber = successfulTasks + failedTasks + 1; taskNumber <= totalTasksCount; i++, taskNumber++) { + DummyTaskFailure failure = failures.get(i); + assertThat(failure.getNumber()).isEqualTo(taskNumber); + assertThat(failure.getError()).isEqualTo("error"); + assertThat(failure.isFailAlways()).isTrue(); + } + }); + } + + private Job submitJob(DummyJobConfiguration configuration) { + return submitJob(configuration, "test-job"); + } + + @SneakyThrows + private Job submitJob(DummyJobConfiguration configuration, String key) { + return jobManager.submitJob(Job.builder() + .tenantId(tenantId) + .type(JobType.DUMMY) + .key(key) + .entityId(jobEntity.getId()) + .configuration(configuration) + .build()).get(); + } + + private List getFailures(JobResult jobResult) { + return jobResult.getResults().stream() + .map(taskResult -> ((DummyTaskResult) taskResult).getFailure()) + .sorted(Comparator.comparingInt(DummyTaskFailure::getNumber)) + .toList(); + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest_EntityPartitioningStrategy.java b/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest_EntityPartitioningStrategy.java new file mode 100644 index 0000000000..59093ad802 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest_EntityPartitioningStrategy.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.job; + +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.dao.service.DaoSqlTest; + +@DaoSqlTest +@TestPropertySource(properties = { + "queue.tasks.stats.processing_interval=0", + "queue.tasks.partitioning_strategy=entity", + "queue.tasks.partitions_per_type=DUMMY:100;DUMMY:50" +}) +public class JobManagerTest_EntityPartitioningStrategy extends JobManagerTest { + + /* + * Some tests are overridden because they are based on + * tenant partitioning strategy (subsequent tasks processing within a tenant) + * */ + + @Override + public void testCancelJob_simulateTaskProcessorRestart() throws Exception { + } + + @Override + public void testSubmitJob_generalError() { + + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/job/TestTaskProcessor.java b/application/src/test/java/org/thingsboard/server/service/job/TestTaskProcessor.java new file mode 100644 index 0000000000..fdaec648b5 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/job/TestTaskProcessor.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.job; + +import org.springframework.stereotype.Component; +import org.thingsboard.server.service.job.task.DummyTaskProcessor; + +@Component +public class TestTaskProcessor extends DummyTaskProcessor { +} diff --git a/application/src/test/java/org/thingsboard/server/service/notification/AbstractNotificationApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/AbstractNotificationApiTest.java index b7e4013bb9..f238a1bbc1 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/AbstractNotificationApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/AbstractNotificationApiTest.java @@ -21,7 +21,6 @@ import org.junit.After; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.util.Pair; -import org.springframework.jdbc.core.JdbcTemplate; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.notification.SlackService; import org.thingsboard.server.common.data.User; @@ -89,8 +88,6 @@ public abstract class AbstractNotificationApiTest extends AbstractControllerTest protected SqlPartitioningRepository partitioningRepository; @Autowired protected DefaultNotifications defaultNotifications; - @Autowired - private JdbcTemplate jdbcTemplate; public static final String DEFAULT_NOTIFICATION_SUBJECT = "Just a test"; public static final NotificationType DEFAULT_NOTIFICATION_TYPE = NotificationType.GENERAL; @@ -101,7 +98,6 @@ public abstract class AbstractNotificationApiTest extends AbstractControllerTest notificationRuleService.deleteNotificationRulesByTenantId(TenantId.SYS_TENANT_ID); notificationTemplateService.deleteNotificationTemplatesByTenantId(TenantId.SYS_TENANT_ID); notificationTargetService.deleteNotificationTargetsByTenantId(TenantId.SYS_TENANT_ID); - jdbcTemplate.execute("TRUNCATE TABLE notification"); partitioningRepository.cleanupPartitionsCache("notification", Long.MAX_VALUE, 0); notificationSettingsService.deleteNotificationSettings(TenantId.SYS_TENANT_ID); } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index aed6eb4cf5..6da6fcf8a8 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -87,6 +87,8 @@ public interface TbClusterService extends TbQueueClusterService { void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + void broadcast(ComponentLifecycleMsg componentLifecycleMsg); + void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); void onDeviceProfileDelete(DeviceProfile deviceProfile, TbQueueCallback callback); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java index e15d9c8ace..99523bf21f 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue; - public interface TbQueueCallback { TbQueueCallback EMPTY = new TbQueueCallback() { @@ -34,4 +33,5 @@ public interface TbQueueCallback { void onSuccess(TbQueueMsgMetadata metadata); void onFailure(Throwable t); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index 65db0d5a76..9adf703e0b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.entity; +import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -25,7 +26,9 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; +import java.util.Map; import java.util.Optional; +import java.util.Set; public interface EntityService { @@ -37,6 +40,8 @@ public interface EntityService { Optional> fetchEntity(TenantId tenantId, EntityId entityId); + Map fetchEntityInfos(TenantId tenantId, CustomerId customerId, Set entityIds); + Optional fetchNameLabelAndCustomerDetails(TenantId tenantId, EntityId entityId); long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/job/JobService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/job/JobService.java new file mode 100644 index 0000000000..4f3f9548d8 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/job/JobService.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.job; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobFilter; +import org.thingsboard.server.common.data.job.JobStats; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.EntityDaoService; + +public interface JobService extends EntityDaoService { + + Job saveJob(TenantId tenantId, Job job); + + Job findJobById(TenantId tenantId, JobId jobId); + + void cancelJob(TenantId tenantId, JobId jobId); + + void markAsFailed(TenantId tenantId, JobId jobId, String error); + + void processStats(TenantId tenantId, JobId jobId, JobStats jobStats); + + PageData findJobsByFilter(TenantId tenantId, JobFilter filter, PageLink pageLink); + + Job findLatestJobByKey(TenantId tenantId, String key); + + void deleteJob(TenantId tenantId, JobId jobId); + + int deleteJobsByEntityId(TenantId tenantId, EntityId entityId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index 7582b377f0..af5fec1827 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -64,7 +64,8 @@ public enum EntityType { MOBILE_APP(37), MOBILE_APP_BUNDLE(38), CALCULATED_FIELD(39), - CALCULATED_FIELD_LINK(40); + CALCULATED_FIELD_LINK(40), + JOB(41); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index ed02d22a74..2d88cef037 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -85,6 +85,10 @@ public class HousekeeperTask implements Serializable { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_CALCULATED_FIELDS); } + public static HousekeeperTask deleteJobs(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_JOBS); + } + @JsonIgnore public String getDescription() { return taskType.getDescription() + " for " + entityId.getEntityType().getNormalName().toLowerCase() + " " + entityId.getId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java index ef217debc3..e84642b599 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -31,7 +31,8 @@ public enum HousekeeperTaskType { UNASSIGN_ALARMS("alarms unassigning"), DELETE_TENANT_ENTITIES("tenant entities deletion"), DELETE_ENTITIES("entities deletion"), - DELETE_CALCULATED_FIELDS("calculated fields deletion"); + DELETE_CALCULATED_FIELDS("calculated fields deletion"), + DELETE_JOBS("jobs deletion"); private final String description; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index f5dd4b12a0..dcf59a4ea4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -117,6 +117,8 @@ public class EntityIdFactory { return new CalculatedFieldId(uuid); case CALCULATED_FIELD_LINK: return new CalculatedFieldLinkId(uuid); + case JOB: + return new JobId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/JobId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/JobId.java new file mode 100644 index 0000000000..76678b8b31 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/JobId.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +public class JobId extends UUIDBased implements EntityId { + + @JsonCreator + public JobId(@JsonProperty("id") UUID id) { + super(id); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "JOB", allowableValues = "JOB") + @Override + public EntityType getEntityType() { + return EntityType.JOB; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/DummyJobConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/DummyJobConfiguration.java new file mode 100644 index 0000000000..a9ff9e7f4f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/DummyJobConfiguration.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; + +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +@AllArgsConstructor +@NoArgsConstructor +@Builder +@ToString(callSuper = true) +public class DummyJobConfiguration extends JobConfiguration { + + private long taskProcessingTimeMs; + private int successfulTasksCount; + private int failedTasksCount; + private int permanentlyFailedTasksCount; + private List errors; + private int retries; + + private String generalError; + private int submittedTasksBeforeGeneralError; + + @Override + public JobType getType() { + return JobType.DUMMY; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/DummyJobResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/DummyJobResult.java new file mode 100644 index 0000000000..031a733d51 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/DummyJobResult.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job; + +public class DummyJobResult extends JobResult { + + @Override + public JobType getJobType() { + return JobType.DUMMY; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/Job.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/Job.java new file mode 100644 index 0000000000..4af60bbd5e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/Job.java @@ -0,0 +1,85 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Set; +import java.util.UUID; + +@Data +@NoArgsConstructor +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) +public class Job extends BaseData implements HasTenantId { + + @NotNull + private TenantId tenantId; + @NotNull + private JobType type; + @NotBlank + private String key; + @NotNull + private EntityId entityId; + private String entityName; // read-only + @NotNull + private JobStatus status; + @NotNull + @Valid + private JobConfiguration configuration; + @NotNull + private JobResult result; + + public static final Set SUPPORTED_ENTITY_TYPES = Set.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE + ); + + @Builder(toBuilder = true) + public Job(TenantId tenantId, JobType type, String key, EntityId entityId, JobConfiguration configuration) { + this.tenantId = tenantId; + this.type = type; + this.key = key; + this.entityId = entityId; + this.configuration = configuration; + this.configuration.setTasksKey(UUID.randomUUID().toString()); + presetResult(); + } + + public void presetResult() { + this.result = switch (type) { + case DUMMY -> new DummyJobResult(); + }; + } + + @SuppressWarnings("unchecked") + public C getConfiguration() { + return (C) configuration; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/JobConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobConfiguration.java new file mode 100644 index 0000000000..54d8166a5d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobConfiguration.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; +import org.thingsboard.server.common.data.job.task.TaskResult; + +import java.io.Serializable; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "DUMMY", value = DummyJobConfiguration.class), +}) +@Data +public abstract class JobConfiguration implements Serializable { + + @NotBlank + private String tasksKey; // internal + private List toReprocess; // internal + + @JsonIgnore + public abstract JobType getType(); + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobFilter.java similarity index 58% rename from edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java rename to common/data/src/main/java/org/thingsboard/server/common/data/job/JobFilter.java index 1f1152af68..1a678ba34b 100644 --- a/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobFilter.java @@ -13,21 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.edqs; +package org.thingsboard.server.common.data.job; -import org.springframework.stereotype.Service; -import org.thingsboard.server.queue.discovery.QueueRoutingInfo; -import org.thingsboard.server.queue.discovery.QueueRoutingInfoService; +import lombok.Builder; +import lombok.Data; -import java.util.Collections; import java.util.List; +import java.util.UUID; -@Service -public class DummyQueueRoutingInfoService implements QueueRoutingInfoService { +@Data +@Builder +public class JobFilter { - @Override - public List getAllQueuesRoutingInfo() { - return Collections.emptyList(); - } + private final List types; + private final List statuses; + private final List entities; + private final Long startTime; + private final Long endTime; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/JobResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobResult.java new file mode 100644 index 0000000000..bd2c73d0d8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobResult.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.job.task.TaskResult; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "jobType") +@JsonSubTypes({ + @Type(name = "DUMMY", value = DummyJobResult.class) +}) +@Data +@NoArgsConstructor +public abstract class JobResult implements Serializable { + + private int successfulCount; + private int failedCount; + private int discardedCount; + private Integer totalCount = null; // set when all tasks are submitted + private List results = new ArrayList<>(); + private String generalError; + + private long startTs; + private long finishTs; + private long cancellationTs; + + @JsonIgnore + public int getCompletedCount() { + return successfulCount + failedCount + discardedCount; + } + + public void processTaskResult(TaskResult taskResult) { + if (taskResult.isSuccess()) { + successfulCount++; + } else if (taskResult.isDiscarded()) { + discardedCount++; + } else { + failedCount++; + if (results.size() < 100) { // preserving only first 100 errors, not reprocessing if there are more failures + results.add(taskResult); + } + } + } + + @JsonIgnore + public abstract JobType getJobType(); + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobStats.java similarity index 60% rename from edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java rename to common/data/src/main/java/org/thingsboard/server/common/data/job/JobStats.java index 4e16e5e16a..50a3b1d759 100644 --- a/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobStats.java @@ -13,18 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.edqs; +package org.thingsboard.server.common.data.job; -import org.springframework.stereotype.Service; +import lombok.Data; +import org.thingsboard.server.common.data.id.JobId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.queue.discovery.TenantRoutingInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; +import org.thingsboard.server.common.data.job.task.TaskResult; -@Service -public class DummyTenantRoutingInfoService implements TenantRoutingInfoService { - @Override - public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { - return null; - } +import java.util.ArrayList; +import java.util.List; + +@Data +public class JobStats { + + private final TenantId tenantId; + private final JobId jobId; + private final List taskResults = new ArrayList<>(); + private Integer totalTasksCount; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/JobStatus.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobStatus.java new file mode 100644 index 0000000000..5a4a9e35ee --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobStatus.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job; + +public enum JobStatus { + + QUEUED, + PENDING, + RUNNING, + COMPLETED, + FAILED, + CANCELLED; + + public boolean isOneOf(JobStatus... statuses) { + if (statuses == null) { + return false; + } + for (JobStatus status : statuses) { + if (this == status) { + return true; + } + } + return false; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/JobType.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobType.java new file mode 100644 index 0000000000..c2c461d12b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/JobType.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +@Getter +public enum JobType { + + DUMMY("Dummy job"); + + private final String title; + + public String getTasksTopic() { + return "tasks." + name().toLowerCase(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/task/DummyTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/task/DummyTask.java new file mode 100644 index 0000000000..e0f670ad9e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/task/DummyTask.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job.task; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.job.JobType; + +import java.util.List; +import java.util.UUID; + +@Data +@NoArgsConstructor +@EqualsAndHashCode(callSuper = true) +@SuperBuilder +@ToString(callSuper = true) +public class DummyTask extends Task { + + private int number; + private long processingTimeMs; + private List errors; // errors for each attempt + private boolean failAlways; + + @Override + public DummyTaskResult toFailed(Throwable error) { + return DummyTaskResult.failed(this, error); + } + + @Override + public DummyTaskResult toDiscarded() { + return DummyTaskResult.discarded(this); + } + + @Override + public EntityId getEntityId() { + return new DeviceId(UUID.randomUUID()); + } + + @Override + public JobType getJobType() { + return JobType.DUMMY; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/task/DummyTaskResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/task/DummyTaskResult.java new file mode 100644 index 0000000000..1988f13eb0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/task/DummyTaskResult.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job.task; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.job.JobType; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@SuperBuilder +@ToString(callSuper = true) +public class DummyTaskResult extends TaskResult { + + private DummyTaskFailure failure; + + public static DummyTaskResult success(DummyTask task) { + return DummyTaskResult.builder() + .key(task.getKey()) + .success(true) + .build(); + } + + public static DummyTaskResult failed(DummyTask task, Throwable error) { + return DummyTaskResult.builder() + .key(task.getKey()) + .failure(DummyTaskFailure.builder() + .error(error.getMessage()) + .number(task.getNumber()) + .failAlways(task.isFailAlways()) + .build()) + .build(); + } + + public static DummyTaskResult discarded(DummyTask task) { + return DummyTaskResult.builder() + .key(task.getKey()) + .discarded(true) + .build(); + } + + @Override + public JobType getJobType() { + return JobType.DUMMY; + } + + @Data + @NoArgsConstructor + @EqualsAndHashCode(callSuper = true) + @SuperBuilder + public static class DummyTaskFailure extends TaskFailure { + + private int number; + private boolean failAlways; + + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/task/Task.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/task/Task.java new file mode 100644 index 0000000000..fb373d9850 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/task/Task.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job.task; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.JobType; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "jobType") +@JsonSubTypes({ + @Type(name = "DUMMY", value = DummyTask.class) +}) +@SuperBuilder +@AllArgsConstructor +public abstract class Task { + + private TenantId tenantId; + private JobId jobId; + private String key; + private int retries; + + public Task() { + } + + private int attempt = 0; + + public abstract R toFailed(Throwable error); + + public abstract R toDiscarded(); + + @JsonIgnore + public abstract EntityId getEntityId(); + + @JsonIgnore + public abstract JobType getJobType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/task/TaskFailure.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/task/TaskFailure.java new file mode 100644 index 0000000000..ce22a08269 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/task/TaskFailure.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job.task; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@SuperBuilder +public abstract class TaskFailure { + + private String error; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/job/task/TaskResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/job/task/TaskResult.java new file mode 100644 index 0000000000..21303a55fe --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/job/task/TaskResult.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.job.task; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.job.JobType; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@SuperBuilder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "jobType") +@JsonSubTypes({ + @Type(name = "DUMMY", value = DummyTaskResult.class) +}) +public abstract class TaskResult { + + private String key; + private boolean success; + private boolean discarded; + + @JsonIgnore + public abstract JobType getJobType(); + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 13fd6159fc..d57301fd10 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg.plugin; +import com.fasterxml.jackson.databind.JsonNode; import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.EntityType; @@ -45,13 +46,14 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final String name; private final EntityId oldProfileId; private final EntityId profileId; + private final JsonNode info; public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this(tenantId, entityId, event, null, null, null, null); + this(tenantId, entityId, event, null, null, null, null, null); } @Builder - private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId) { + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, JsonNode info) { this.tenantId = tenantId; this.entityId = entityId; this.event = event; @@ -59,6 +61,7 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { this.name = name; this.oldProfileId = oldProfileId; this.profileId = profileId; + this.info = info; } public Optional getRuleChainId() { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index f31fdfa7a8..8fd535891c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -27,7 +27,8 @@ public enum ServiceType { TB_TRANSPORT("TB Transport"), JS_EXECUTOR("JS Executor"), TB_VC_EXECUTOR("TB VC Executor"), - EDQS("TB Entity Data Query Service"); + EDQS("TB Entity Data Query Service"), + TASK_PROCESSOR("Task Processor"); private final String label; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java index ee8990d931..777ce3bf94 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg.queue; +import com.google.common.util.concurrent.SettableFuture; import org.thingsboard.server.common.data.id.EntityId; import java.util.UUID; @@ -34,7 +35,7 @@ public interface TbCallback { } }; - default UUID getId(){ + default UUID getId() { return EntityId.NULL_UUID; } @@ -42,4 +43,18 @@ public interface TbCallback { void onFailure(Throwable t); + static TbCallback wrap(SettableFuture future) { + return new TbCallback() { + @Override + public void onSuccess() { + future.set(null); + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index bb2e0fb2d1..bc96952ca7 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -135,6 +135,9 @@ public class ProtoUtils { if (msg.getOldName() != null) { builder.setOldName(msg.getOldName()); } + if (msg.getInfo() != null) { + builder.setInfo(JacksonUtil.toString(msg.getInfo())); + } return builder.build(); } @@ -162,6 +165,9 @@ public class ProtoUtils { var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); } + if (proto.hasInfo()) { + builder.info(JacksonUtil.toJsonNode(proto.getInfo())); + } return builder.build(); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 65694f2fb6..c413ecb3c1 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -63,6 +63,7 @@ enum EntityTypeProto { MOBILE_APP_BUNDLE = 38; CALCULATED_FIELD = 39; CALCULATED_FIELD_LINK = 40; + JOB = 41; } enum ApiUsageRecordKeyProto { @@ -90,6 +91,7 @@ message ServiceInfo { repeated string assignedTenantProfiles = 11; string label = 12; bool ready = 13; + repeated string taskTypes = 14; } message SystemInfoProto { @@ -1263,6 +1265,7 @@ message ComponentLifecycleMsgProto { int64 oldProfileIdLSB = 10; int64 profileIdMSB = 11; int64 profileIdLSB = 12; + optional string info = 13; } message EdgeEventMsgProto { @@ -1850,3 +1853,20 @@ message EdqsRequestMsg { message EdqsResponseMsg { string value = 1; } + +message TaskProto { + string value = 1; +} + +message JobStatsMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 jobIdMSB = 3; + int64 jobIdLSB = 4; + optional TaskResultProto taskResult = 5; + optional int32 totalTasksCount = 6; +} + +message TaskResultProto { + string value = 1; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueConsumerManager.java index ffed499d8d..5025d887cb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueConsumerManager.java @@ -25,7 +25,11 @@ import org.thingsboard.server.queue.TbQueueMsg; import java.util.List; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; @Slf4j @@ -39,6 +43,7 @@ public class QueueConsumerManager { @Getter private final TbQueueConsumer consumer; + private Future consumerTask; private volatile boolean stopped; @Builder @@ -63,7 +68,7 @@ public class QueueConsumerManager { public void launch() { log.info("[{}] Launching consumer", name); - consumerExecutor.submit(() -> { + consumerTask = consumerExecutor.submit(() -> { if (threadPrefix != null) { ThingsBoardThreadFactory.addThreadNamePrefix(threadPrefix); } @@ -101,6 +106,13 @@ public class QueueConsumerManager { log.debug("[{}] Stopping consumer", name); stopped = true; consumer.unsubscribe(); + try { + if (consumerTask != null) { + consumerTask.get(10, TimeUnit.SECONDS); + } + } catch (InterruptedException | ExecutionException | TimeoutException e) { + log.error("[{}] Failed to await consumer loop stop", name, e); + } } public interface MsgPackProcessor { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index c2705cb1ab..3c0b6f5b51 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -24,11 +24,13 @@ import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbTransportService; +import org.thingsboard.server.common.data.job.JobType; import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.task.TaskProcessor; import org.thingsboard.server.queue.util.AfterContextReady; import java.net.InetAddress; @@ -40,7 +42,12 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import static org.thingsboard.common.util.SystemUtil.*; +import static org.thingsboard.common.util.SystemUtil.getCpuCount; +import static org.thingsboard.common.util.SystemUtil.getCpuUsage; +import static org.thingsboard.common.util.SystemUtil.getDiscSpaceUsage; +import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; +import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; +import static org.thingsboard.common.util.SystemUtil.getTotalMemory; @Component @@ -59,13 +66,17 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.rule_engine.assigned_tenant_profiles:}") private Set assignedTenantProfiles; - @Autowired + @Autowired(required = false) private EdqsConfig edqsConfig; @Autowired private ApplicationContext applicationContext; + @Autowired(required = false) + private List> availableTaskProcessors; + private List serviceTypes; + private List taskTypes; private ServiceInfo serviceInfo; private boolean ready = true; @@ -94,6 +105,13 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { edqsConfig.setLabel(serviceId); } } + if (CollectionsUtil.isNotEmpty(availableTaskProcessors)) { + taskTypes = availableTaskProcessors.stream() + .map(TaskProcessor::getJobType) + .toList(); + } else { + taskTypes = Collections.emptyList(); + } generateNewServiceInfoWithCurrentSystemInfo(); } @@ -130,8 +148,11 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (CollectionsUtil.isNotEmpty(assignedTenantProfiles)) { builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList())); } - builder.setLabel(edqsConfig.getLabel()); + if (edqsConfig != null) { + builder.setLabel(edqsConfig.getLabel()); + } builder.setReady(ready); + builder.addAllTaskTypes(taskTypes.stream().map(JobType::name).toList()); return serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index ec5c675e32..47a6732578 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -19,6 +19,7 @@ import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import jakarta.annotation.PostConstruct; import lombok.Data; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; @@ -29,6 +30,7 @@ import org.thingsboard.server.common.data.exception.TenantNotFoundException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.job.JobType; import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -38,16 +40,19 @@ import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.PropertyUtils; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -62,6 +67,7 @@ import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; @Service @Slf4j +@RequiredArgsConstructor public class HashPartitionService implements PartitionService { @Value("${queue.core.topic:tb_core}") @@ -82,13 +88,17 @@ public class HashPartitionService implements PartitionService { private Integer edgePartitions; @Value("${queue.edqs.partitions:12}") private Integer edqsPartitions; + @Value("${queue.tasks.partitions:12}") + private Integer defaultTasksPartitions; + @Value("${queue.tasks.partitions_per_type:}") + private String tasksPartitionsPerType; @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; - private final TenantRoutingInfoService tenantRoutingInfoService; - private final QueueRoutingInfoService queueRoutingInfoService; + private final Optional tenantRoutingInfoService; + private final Optional queueRoutingInfoService; private final TopicService topicService; protected volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); @@ -105,18 +115,6 @@ public class HashPartitionService implements PartitionService { private HashFunction hashFunction; - public HashPartitionService(TbServiceInfoProvider serviceInfoProvider, - TenantRoutingInfoService tenantRoutingInfoService, - ApplicationEventPublisher applicationEventPublisher, - QueueRoutingInfoService queueRoutingInfoService, - TopicService topicService) { - this.serviceInfoProvider = serviceInfoProvider; - this.tenantRoutingInfoService = tenantRoutingInfoService; - this.applicationEventPublisher = applicationEventPublisher; - this.queueRoutingInfoService = queueRoutingInfoService; - this.topicService = topicService; - } - @PostConstruct public void init() { this.hashFunction = forName(hashFunctionName); @@ -140,6 +138,19 @@ public class HashPartitionService implements PartitionService { QueueKey edqsKey = new QueueKey(ServiceType.EDQS); partitionSizesMap.put(edqsKey, edqsPartitions); partitionTopicsMap.put(edqsKey, "edqs"); // placeholder, not used + + Map tasksPartitions = new EnumMap<>(JobType.class); + PropertyUtils.getProps(tasksPartitionsPerType).forEach((type, partitions) -> { + tasksPartitions.put(JobType.valueOf(type), Integer.parseInt(partitions)); + }); + for (JobType type : JobType.values()) { + tasksPartitions.putIfAbsent(type, defaultTasksPartitions); + } + tasksPartitions.forEach((type, partitions) -> { + QueueKey queueKey = new QueueKey(ServiceType.TASK_PROCESSOR, type.name()); + partitionSizesMap.put(queueKey, partitions); + partitionTopicsMap.put(queueKey, type.getTasksTopic()); + }); } @AfterStartUp(order = AfterStartUp.QUEUE_INFO_INITIALIZATION) @@ -169,6 +180,10 @@ public class HashPartitionService implements PartitionService { } private List getQueueRoutingInfos() { + if (queueRoutingInfoService.isEmpty()) { + return Collections.emptyList(); + } + List queueRoutingInfoList; String serviceType = serviceInfoProvider.getServiceType(); @@ -179,7 +194,7 @@ public class HashPartitionService implements PartitionService { if (getQueuesRetries > 0) { log.info("Try to get queue routing info."); try { - queueRoutingInfoList = queueRoutingInfoService.getAllQueuesRoutingInfo(); + queueRoutingInfoList = queueRoutingInfoService.get().getAllQueuesRoutingInfo(); break; } catch (Exception e) { log.info("Failed to get queues routing info: {}!", e.getMessage()); @@ -195,7 +210,7 @@ public class HashPartitionService implements PartitionService { } } } else { - queueRoutingInfoList = queueRoutingInfoService.getAllQueuesRoutingInfo(); + queueRoutingInfoList = queueRoutingInfoService.get().getAllQueuesRoutingInfo(); } return queueRoutingInfoList; } @@ -454,8 +469,8 @@ public class HashPartitionService implements PartitionService { if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { partitionSizesMap.keySet().stream() .filter(queueKey -> queueKey.getType() == ServiceType.TB_RULE_ENGINE && - !queueKey.getTenantId().isSysTenantId() && - !newPartitions.containsKey(queueKey)) + !queueKey.getTenantId().isSysTenantId() && + !newPartitions.containsKey(queueKey)) .forEach(removed::add); } removed.forEach(queueKey -> { @@ -629,7 +644,11 @@ public class HashPartitionService implements PartitionService { } private TenantRoutingInfo getRoutingInfo(TenantId tenantId) { - return tenantRoutingInfoMap.computeIfAbsent(tenantId, tenantRoutingInfoService::getRoutingInfo); + if (tenantRoutingInfoService.isPresent()) { + return tenantRoutingInfoMap.computeIfAbsent(tenantId, __ -> tenantRoutingInfoService.get().getRoutingInfo(tenantId)); + } else { + return new TenantRoutingInfo(tenantId, null, false); + } } protected TenantId getIsolatedOrSystemTenantId(ServiceType serviceType, TenantId tenantId) { @@ -675,6 +694,10 @@ public class HashPartitionService implements PartitionService { for (String transportType : instance.getTransportsList()) { tbTransportServicesByType.computeIfAbsent(transportType, t -> new ArrayList<>()).add(instance); } + for (String taskType : instance.getTaskTypesList()) { + QueueKey queueKey = new QueueKey(ServiceType.TASK_PROCESSOR, taskType); + queueServiceList.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(instance); + } } @NotNull @@ -689,7 +712,7 @@ public class HashPartitionService implements PartitionService { if (!responsibleServices.isEmpty()) { // if there are any dedicated servers TenantProfileId profileId; if (tenantId != null && !tenantId.isSysTenantId()) { - TenantRoutingInfo routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); + TenantRoutingInfo routingInfo = tenantRoutingInfoService.get().getRoutingInfo(tenantId); profileId = routingInfo.getProfileId(); } else { profileId = null; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index aebda5a5bc..5d5834d20a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -62,6 +62,8 @@ public class TbKafkaTopicConfigs { private String edqsRequestsProperties; @Value("${queue.kafka.topic-properties.edqs-state:}") private String edqsStateProperties; + @Value("${queue.kafka.topic-properties.tasks:}") + private String tasksProperties; @Getter private Map coreConfigs; @@ -99,6 +101,8 @@ public class TbKafkaTopicConfigs { private Map edqsRequestsConfigs; @Getter private Map edqsStateConfigs; + @Getter + private Map tasksConfigs; @PostConstruct private void init() { @@ -122,6 +126,7 @@ public class TbKafkaTopicConfigs { edqsEventsConfigs = PropertyUtils.getProps(edqsEventsProperties); edqsRequestsConfigs = PropertyUtils.getProps(edqsRequestsProperties); edqsStateConfigs = PropertyUtils.getProps(edqsStateProperties); + tasksConfigs = PropertyUtils.getProps(tasksProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 085d04f28c..d3233f8dee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -27,6 +27,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.JobStatsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; @@ -41,6 +42,7 @@ import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TasksQueueConfig; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; @@ -66,6 +68,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueEdgeSettings edgeSettings; private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final EdqsConfig edqsConfig; + private final TasksQueueConfig tasksQueueConfig; private final InMemoryStorage storage; @Override @@ -258,9 +261,13 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE .build(); } + @Override + public TbQueueConsumer> createJobStatsConsumer() { + return new InMemoryTbQueueConsumer<>(storage, tasksQueueConfig.getStatsTopic()); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 2269004f90..866f8d235e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.JobStatsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -63,6 +64,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TasksQueueConfig; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; @@ -92,6 +94,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final EdqsConfig edqsConfig; + private final TasksQueueConfig tasksQueueConfig; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -110,6 +113,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin cfStateAdmin; private final TbQueueAdmin edqsEventsAdmin; private final TbKafkaAdmin edqsRequestsAdmin; + private final TbQueueAdmin tasksAdmin; private final AtomicLong consumerCount = new AtomicLong(); private final AtomicLong edgeConsumerCount = new AtomicLong(); @@ -126,7 +130,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs, - EdqsConfig edqsConfig) { + EdqsConfig edqsConfig, + TasksQueueConfig tasksQueueConfig) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -140,6 +145,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.edgeSettings = edgeSettings; this.calculatedFieldSettings = calculatedFieldSettings; this.edqsConfig = edqsConfig; + this.tasksQueueConfig = tasksQueueConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -158,6 +164,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); + this.tasksAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTasksConfigs()); } @Override @@ -641,6 +648,19 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi .build(); } + @Override + public TbQueueConsumer> createJobStatsConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(tasksQueueConfig.getStatsTopic())) + .clientId("job-stats-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("job-stats-consumer-group")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), JobStatsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(tasksAdmin) + .statsService(consumerStatsService) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index ea7c56f0aa..70009aa29d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.JobStatsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -59,6 +60,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TasksQueueConfig; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; @@ -88,6 +90,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueEdgeSettings edgeSettings; private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final EdqsConfig edqsConfig; + private final TasksQueueConfig tasksQueueConfig; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -105,6 +108,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin cfAdmin; private final TbQueueAdmin edqsEventsAdmin; private final TbKafkaAdmin edqsRequestsAdmin; + private final TbQueueAdmin tasksAdmin; private final AtomicLong consumerCount = new AtomicLong(); private final AtomicLong edgeConsumerCount = new AtomicLong(); @@ -122,6 +126,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueCalculatedFieldSettings calculatedFieldSettings, EdqsConfig edqsConfig, + TasksQueueConfig tasksQueueConfig, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -136,6 +141,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.edgeSettings = edgeSettings; this.calculatedFieldSettings = calculatedFieldSettings; this.edqsConfig = edqsConfig; + this.tasksQueueConfig = tasksQueueConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -153,6 +159,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); + this.tasksAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTasksConfigs()); } @Override @@ -520,6 +527,19 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { .build(); } + @Override + public TbQueueConsumer> createJobStatsConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(tasksQueueConfig.getStatsTopic())) + .clientId("job-stats-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("job-stats-consumer-group")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), JobStatsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(tasksAdmin) + .statsService(consumerStatsService) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index b4884ae72c..3b67ea4f9f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -438,4 +438,5 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { cfAdmin.destroy(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 037d1f2087..37c15d5b87 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.JobStatsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -165,4 +166,6 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); + TbQueueConsumer> createJobStatsConsumer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index a7a34992cd..cb7e6dd1f4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -121,4 +121,5 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TasksQueueConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TasksQueueConfig.java new file mode 100644 index 0000000000..7c94596139 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TasksQueueConfig.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Getter +@Component +public class TasksQueueConfig { + + @Value("${queue.tasks.poll_interval:500}") + private int pollInterval; + + @Value("${queue.tasks.partitioning_strategy:tenant}") + private String partitioningStrategy; + + @Value("${queue.tasks.stats.topic:jobs.stats}") + private String statsTopic; + + @Value("${queue.tasks.stats.poll_interval:500}") + private int statsPollInterval; + + @Value("${queue.tasks.stats.processing_interval:1000}") + private int statsProcessingInterval; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/task/InMemoryTaskProcessorQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/task/InMemoryTaskProcessorQueueFactory.java new file mode 100644 index 0000000000..76effa6304 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/task/InMemoryTaskProcessorQueueFactory.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.task; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.gen.transport.TransportProtos.JobStatsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.memory.InMemoryStorage; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TasksQueueConfig; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") +@RequiredArgsConstructor +public class InMemoryTaskProcessorQueueFactory implements TaskProcessorQueueFactory { + + private final InMemoryStorage storage; + private final TasksQueueConfig tasksQueueConfig; + + @Override + public TbQueueConsumer> createTaskConsumer(JobType jobType) { + return new InMemoryTbQueueConsumer<>(storage, jobType.getTasksTopic()); + } + + @Override + public TbQueueProducer> createJobStatsProducer() { + return new InMemoryTbQueueProducer<>(storage, tasksQueueConfig.getStatsTopic()); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/task/InMemoryTaskProducerQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/task/InMemoryTaskProducerQueueFactory.java new file mode 100644 index 0000000000..7f0cae7eb1 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/task/InMemoryTaskProducerQueueFactory.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.task; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.memory.InMemoryStorage; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; + +@Component +@ConditionalOnExpression("'${queue.type:null}' == 'in-memory'") +@RequiredArgsConstructor +public class InMemoryTaskProducerQueueFactory implements TaskProducerQueueFactory { + + private final InMemoryStorage storage; + + @Override + public TbQueueProducer> createTaskProducer(JobType jobType) { + return new InMemoryTbQueueProducer<>(storage, jobType.getTasksTopic()); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/task/JobStatsService.java b/common/queue/src/main/java/org/thingsboard/server/queue/task/JobStatsService.java new file mode 100644 index 0000000000..0b7d18fde4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/task/JobStatsService.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.task; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.task.TaskResult; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.JobStatsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TaskResultProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +@Lazy +@Service +@Slf4j +public class JobStatsService { + + private final TbQueueProducer> producer; + + public JobStatsService(TaskProcessorQueueFactory queueFactory) { + this.producer = queueFactory.createJobStatsProducer(); + } + + public void reportTaskResult(TenantId tenantId, JobId jobId, TaskResult result) { + report(tenantId, jobId, JobStatsMsg.newBuilder() + .setTaskResult(TaskResultProto.newBuilder() + .setValue(JacksonUtil.toString(result)) + .build())); + } + + public void reportAllTasksSubmitted(TenantId tenantId, JobId jobId, int tasksCount) { + report(tenantId, jobId, JobStatsMsg.newBuilder() + .setTotalTasksCount(tasksCount)); + } + + private void report(TenantId tenantId, JobId jobId, JobStatsMsg.Builder statsMsg) { + log.debug("[{}][{}] Reporting: {}", tenantId, jobId, statsMsg); + statsMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setJobIdMSB(jobId.getId().getMostSignificantBits()) + .setJobIdLSB(jobId.getId().getLeastSignificantBits()); + + // using job id as msg key so that all stats for a certain job are submitted to the same partition + TbProtoQueueMsg msg = new TbProtoQueueMsg<>(jobId.getId(), statsMsg.build()); + producer.send(TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(), msg, TbQueueCallback.EMPTY); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/task/KafkaTaskProcessorQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/task/KafkaTaskProcessorQueueFactory.java new file mode 100644 index 0000000000..6b95b0530c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/task/KafkaTaskProcessorQueueFactory.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.task; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.gen.transport.TransportProtos.JobStatsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TasksQueueConfig; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class KafkaTaskProcessorQueueFactory implements TaskProcessorQueueFactory { + + private final TopicService topicService; + private final TbServiceInfoProvider serviceInfoProvider; + private final TasksQueueConfig tasksQueueConfig; + private final TbKafkaSettings kafkaSettings; + private final TbKafkaConsumerStatsService consumerStatsService; + + private final TbQueueAdmin tasksAdmin; + + public KafkaTaskProcessorQueueFactory(TopicService topicService, + TbServiceInfoProvider serviceInfoProvider, + TasksQueueConfig tasksQueueConfig, + TbKafkaSettings kafkaSettings, + TbKafkaConsumerStatsService consumerStatsService, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.topicService = topicService; + this.serviceInfoProvider = serviceInfoProvider; + this.tasksQueueConfig = tasksQueueConfig; + this.kafkaSettings = kafkaSettings; + this.consumerStatsService = consumerStatsService; + this.tasksAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTasksConfigs()); + } + + @Override + public TbQueueConsumer> createTaskConsumer(JobType jobType) { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(jobType.getTasksTopic())) + .clientId(jobType.name().toLowerCase() + "-task-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName(jobType.name().toLowerCase() + "-task-consumer-group")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TaskProto.parseFrom(msg.getData()), msg.getHeaders())) + .admin(tasksAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createJobStatsProducer() { + return TbKafkaProducerTemplate.>builder() + .clientId("job-stats-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(tasksQueueConfig.getStatsTopic())) + .settings(kafkaSettings) + .admin(tasksAdmin) + .build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/task/KafkaTaskProducerQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/task/KafkaTaskProducerQueueFactory.java new file mode 100644 index 0000000000..b19db211fe --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/task/KafkaTaskProducerQueueFactory.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.task; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; + +@Component +@ConditionalOnExpression("'${queue.type:null}' == 'kafka' && ('${service.type:null}' == 'monolith' || " + + "'${service.type:null}' == 'tb-core' || '${service.type:null}' == 'tb-rule-engine')") +public class KafkaTaskProducerQueueFactory implements TaskProducerQueueFactory { + + private final TopicService topicService; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbKafkaSettings kafkaSettings; + private final TbQueueAdmin tasksAdmin; + + KafkaTaskProducerQueueFactory(TopicService topicService, + TbServiceInfoProvider serviceInfoProvider, + TbKafkaSettings kafkaSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.topicService = topicService; + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.tasksAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTasksConfigs()); + } + + @Override + public TbQueueProducer> createTaskProducer(JobType jobType) { + return TbKafkaProducerTemplate.>builder() + .clientId(jobType.name().toLowerCase() + "-task-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(jobType.getTasksTopic())) + .settings(kafkaSettings) + .admin(tasksAdmin) + .build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProcessor.java b/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProcessor.java new file mode 100644 index 0000000000..ef0e48e30a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProcessor.java @@ -0,0 +1,222 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.task; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.event.EventListener; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.SetCache; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.job.task.Task; +import org.thingsboard.server.common.data.job.task.TaskResult; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.settings.TasksQueueConfig; + +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public abstract class TaskProcessor, R extends TaskResult> { + + protected final Logger log = LoggerFactory.getLogger(getClass()); + + @Autowired + private TaskProcessorQueueFactory queueFactory; + @Autowired + private JobStatsService statsService; + @Autowired + private TaskProcessorExecutors executors; + @Autowired + private TasksQueueConfig config; + + private QueueKey queueKey; + private MainQueueConsumerManager, QueueConfig> taskConsumer; + private final ExecutorService taskExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(getJobType().name().toLowerCase() + "-task-processor")); + + private final SetCache discarded = new SetCache<>(TimeUnit.MINUTES.toMillis(60)); + private final SetCache failed = new SetCache<>(TimeUnit.MINUTES.toMillis(60)); + + private final SetCache deletedTenants = new SetCache<>(TimeUnit.MINUTES.toMillis(60)); + + @PostConstruct + public void init() { + queueKey = new QueueKey(ServiceType.TASK_PROCESSOR, getJobType().name()); + taskConsumer = MainQueueConsumerManager., QueueConfig>builder() + .queueKey(queueKey) + .config(QueueConfig.of(true, config.getPollInterval())) + .msgPackProcessor(this::processMsgs) + .consumerCreator((queueConfig, tpi) -> queueFactory.createTaskConsumer(getJobType())) + .consumerExecutor(executors.getConsumersExecutor()) + .scheduler(executors.getScheduler()) + .taskExecutor(executors.getMgmtExecutor()) + .build(); + } + + @EventListener + public void onPartitionChangeEvent(PartitionChangeEvent event) { + if (event.getServiceType() == ServiceType.TASK_PROCESSOR) { + Set partitions = event.getNewPartitions().get(queueKey); + taskConsumer.update(partitions); + } + } + + @EventListener + public void onComponentLifecycle(ComponentLifecycleMsg event) { + EntityId entityId = event.getEntityId(); + switch (entityId.getEntityType()) { + case JOB -> { + String tasksKey = event.getInfo().get("tasksKey").asText(); + if (event.getEvent() == ComponentLifecycleEvent.STOPPED) { + log.info("Adding job {} ({}) to discarded", entityId, tasksKey); + addToDiscarded(tasksKey); + } else if (event.getEvent() == ComponentLifecycleEvent.FAILED) { + log.info("Adding job {} ({}) to failed", entityId, tasksKey); + failed.add(tasksKey); + } + } + case TENANT -> { + if (event.getEvent() == ComponentLifecycleEvent.DELETED) { + deletedTenants.add(entityId.getId()); + log.info("Adding tenant {} to deleted", entityId); + } + } + } + } + + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig queueConfig) throws Exception { + for (TbProtoQueueMsg msg : msgs) { + try { + @SuppressWarnings("unchecked") + T task = (T) JacksonUtil.fromString(msg.getValue().getValue(), Task.class); + if (discarded.contains(task.getKey())) { + log.debug("Skipping task for discarded job {}: {}", task.getJobId(), task); + reportTaskDiscarded(task); + continue; + } else if (failed.contains(task.getKey())) { + log.debug("Skipping task for failed job {}: {}", task.getJobId(), task); + continue; + } else if (deletedTenants.contains(task.getTenantId().getId())) { + log.debug("Skipping task for deleted tenant {}: {}", task.getTenantId(), task); + continue; + } + + processTask(task); + } catch (InterruptedException e) { + throw e; + } catch (Exception e) { + log.error("Failed to process msg: {}", msg, e); + } + } + consumer.commit(); + } + + private void processTask(T task) throws InterruptedException { + task.setAttempt(task.getAttempt() + 1); + log.debug("Processing task: {}", task); + Future future = null; + try { + long startNs = System.nanoTime(); + future = taskExecutor.submit(() -> process(task)); + R result; + try { + result = future.get(getTaskProcessingTimeout(), TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw e.getCause(); + } catch (TimeoutException e) { + throw new TimeoutException("Timeout after " + getTaskProcessingTimeout() + " ms"); + } + long timingNs = System.nanoTime() - startNs; + log.info("Processed task in {} ms: {}", timingNs / 1000000.0, task); + reportTaskResult(task, result); + } catch (InterruptedException e) { + throw e; + } catch (Throwable e) { + log.error("Failed to process task (attempt {}): {}", task.getAttempt(), task, e); + if (task.getAttempt() <= task.getRetries()) { + processTask(task); + } else { + reportTaskFailure(task, e); + } + } finally { + if (future != null && !future.isDone()) { + future.cancel(true); + } + } + } + + public abstract R process(T task) throws Exception; + + private void reportTaskFailure(T task, Throwable error) { + R taskResult = task.toFailed(error); + reportTaskResult(task, taskResult); + } + + private void reportTaskDiscarded(T task) { + R taskResult = task.toDiscarded(); + reportTaskResult(task, taskResult); + } + + private void reportTaskResult(T task, R result) { + statsService.reportTaskResult(task.getTenantId(), task.getJobId(), result); + } + + public void addToDiscarded(String tasksKey) { + discarded.add(tasksKey); + } + + protected V wait(Future future) throws Exception { + try { + return future.get(); // will be interrupted after task processing timeout + } catch (InterruptedException e) { + future.cancel(true); // interrupting the underlying task + throw e; + } + } + + @PreDestroy + public void destroy() { + taskConsumer.stop(); + taskConsumer.awaitStop(); + taskExecutor.shutdownNow(); + } + + public abstract long getTaskProcessingTimeout(); + + public abstract JobType getJobType(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProcessorExecutors.java b/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProcessorExecutors.java new file mode 100644 index 0000000000..3aa6a0f004 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProcessorExecutors.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.task; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +@Getter +@Lazy +@Component +public class TaskProcessorExecutors { + + private ExecutorService consumersExecutor; + private ExecutorService mgmtExecutor; + private ScheduledExecutorService scheduler; + + @PostConstruct + private void init() { + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("task-consumer")); + mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "task-consumer-mgmt"); + scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("task-consumer-scheduler"); + } + + @PreDestroy + private void destroy() { + if (consumersExecutor != null) { + consumersExecutor.shutdownNow(); + } + if (mgmtExecutor != null) { + mgmtExecutor.shutdownNow(); + } + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProcessorQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProcessorQueueFactory.java new file mode 100644 index 0000000000..c5e8035d74 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProcessorQueueFactory.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.task; + +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.gen.transport.TransportProtos.JobStatsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface TaskProcessorQueueFactory { + + TbQueueConsumer> createTaskConsumer(JobType jobType); + + TbQueueProducer> createJobStatsProducer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProducerQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProducerQueueFactory.java new file mode 100644 index 0000000000..ffb64a07ce --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/task/TaskProducerQueueFactory.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.task; + +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface TaskProducerQueueFactory { + + TbQueueProducer> createTaskProducer(JobType jobType); + +} diff --git a/common/util/pom.xml b/common/util/pom.xml index 6719dc628a..f69cf794e9 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -116,6 +116,10 @@ exp4j ${exp4j.version} + + com.github.ben-manes.caffeine + caffeine + diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index 5903c10ac2..d153501b92 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -37,9 +37,10 @@ import com.google.common.collect.Lists; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.Contract; +import org.thingsboard.server.common.data.Views; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.Views; import java.io.File; import java.io.IOException; @@ -109,6 +110,7 @@ public class JacksonUtil { } } + @Contract("null, _ -> null") // so that IDE doesn't show NPE warning when input is not null public static T fromString(String string, Class clazz) { try { return string != null ? OBJECT_MAPPER.readValue(string, clazz) : null; diff --git a/common/util/src/main/java/org/thingsboard/common/util/SetCache.java b/common/util/src/main/java/org/thingsboard/common/util/SetCache.java new file mode 100644 index 0000000000..9676434534 --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/SetCache.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import java.util.concurrent.TimeUnit; + +public class SetCache { + + private static final Object DUMMY_VALUE = Boolean.TRUE; + + private final Cache cache; + + public SetCache(long valueTtlMs) { + this.cache = Caffeine.newBuilder() + .expireAfterWrite(valueTtlMs, TimeUnit.MILLISECONDS) + .build(); + } + + public void add(K key) { + cache.put(key, DUMMY_VALUE); + } + + public boolean contains(K key) { + return cache.asMap().containsKey(key); + } + + public void remove(K key) { + cache.invalidate(key); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 84f972dc02..2fca546fc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasEmail; @@ -41,20 +42,25 @@ import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityNameFilter; import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.msg.edqs.EdqsApiService; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.stats.EdqsStatsService; import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.model.ModelConstants; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -203,6 +209,30 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return fetchAndConvert(tenantId, entityId, Function.identity()); } + @Override + public Map fetchEntityInfos(TenantId tenantId, CustomerId customerId, Set entityIds) { + Map infos = new HashMap<>(); + entityIds.stream() + .collect(Collectors.groupingBy(EntityId::getEntityType)) + .forEach((entityType, ids) -> { + EntityListFilter filter = new EntityListFilter(); + filter.setEntityType(entityType); + filter.setEntityList(ids.stream().map(Object::toString).toList()); + EntityDataQuery query = new EntityDataQuery(filter, new EntityDataPageLink(ids.size(), 0, null, null), + List.of(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.NAME_PROPERTY)), Collections.emptyList(), Collections.emptyList()); + + entityQueryDao.findEntityDataByQuery(tenantId, customerId, query).getData().forEach(entityData -> { + EntityId entityId = entityData.getEntityId(); + Optional.ofNullable(entityData.getLatest().get(EntityKeyType.ENTITY_FIELD)) + .map(fields -> fields.get(ModelConstants.NAME_PROPERTY)) + .map(TsValue::getValue).ifPresent(name -> { + infos.put(entityId, new EntityInfo(entityId, name)); + }); + }); + }); + return infos; + } + private Optional fetchAndConvert(TenantId tenantId, EntityId entityId, Function, T> converter) { EntityDaoService entityDaoService = entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()); Optional> entityOpt = entityDaoService.findEntity(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 1ca2973936..0340cd0fde 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.Job; import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; import org.thingsboard.server.dao.eventsourcing.ActionCause; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; @@ -76,6 +77,9 @@ public class CleanUpService { submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); submitTask(HousekeeperTask.deleteAlarms(tenantId, entityId)); submitTask(HousekeeperTask.deleteCalculatedFields(tenantId, entityId)); + if (Job.SUPPORTED_ENTITY_TYPES.contains(entityId.getEntityType())) { + submitTask(HousekeeperTask.deleteJobs(tenantId, entityId)); + } } public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/job/DefaultJobService.java b/dao/src/main/java/org/thingsboard/server/dao/job/DefaultJobService.java new file mode 100644 index 0000000000..153e95a404 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/job/DefaultJobService.java @@ -0,0 +1,257 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.job; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobFilter; +import org.thingsboard.server.common.data.job.JobResult; +import org.thingsboard.server.common.data.job.JobStats; +import org.thingsboard.server.common.data.job.JobStatus; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.job.task.TaskResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.service.ConstraintValidator; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.thingsboard.server.common.data.job.JobStatus.CANCELLED; +import static org.thingsboard.server.common.data.job.JobStatus.COMPLETED; +import static org.thingsboard.server.common.data.job.JobStatus.FAILED; +import static org.thingsboard.server.common.data.job.JobStatus.PENDING; +import static org.thingsboard.server.common.data.job.JobStatus.QUEUED; +import static org.thingsboard.server.common.data.job.JobStatus.RUNNING; + +@Service +@RequiredArgsConstructor +@Slf4j +public class DefaultJobService extends AbstractEntityService implements JobService { + + private final JobDao jobDao; + private final EntityService entityService; + + @Transactional + @Override + public Job saveJob(TenantId tenantId, Job job) { + if (jobDao.existsByTenantAndKeyAndStatusOneOf(tenantId, job.getKey(), QUEUED, PENDING, RUNNING)) { + throw new IllegalArgumentException("The same job is already queued or running"); + } + if (jobDao.existsByTenantIdAndTypeAndStatusOneOf(tenantId, job.getType(), PENDING, RUNNING)) { + job.setStatus(QUEUED); + } else { + job.setStatus(PENDING); + job.getResult().setStartTs(System.currentTimeMillis()); + } + return saveJob(tenantId, job, true, null); + } + + @Override + public Job findJobById(TenantId tenantId, JobId jobId) { + return jobDao.findById(tenantId, jobId.getId()); + } + + @Transactional + @Override + public void cancelJob(TenantId tenantId, JobId jobId) { + Job job = findForUpdate(tenantId, jobId); + if (!job.getStatus().isOneOf(QUEUED, PENDING, RUNNING)) { + throw new IllegalArgumentException("Job already " + job.getStatus().name().toLowerCase()); + } + job.getResult().setCancellationTs(System.currentTimeMillis()); + JobStatus prevStatus = job.getStatus(); + if (job.getStatus() == QUEUED) { + job.setStatus(CANCELLED); // setting cancelled status right away, because we don't expect stats for cancelled tasks + } else if (job.getStatus() == PENDING) { + job.setStatus(RUNNING); + } + saveJob(tenantId, job, true, prevStatus); + } + + @Transactional + @Override + public void markAsFailed(TenantId tenantId, JobId jobId, String error) { + Job job = findForUpdate(tenantId, jobId); + job.getResult().setGeneralError(error); + JobStatus prevStatus = job.getStatus(); + job.setStatus(FAILED); + saveJob(tenantId, job, true, prevStatus); + } + + @Transactional + @Override + public void processStats(TenantId tenantId, JobId jobId, JobStats jobStats) { + Job job = findForUpdate(tenantId, jobId); + if (job == null) { + log.debug("[{}][{}] Got stale stats: {}", tenantId, jobId, jobStats); + return; + } + JobStatus prevStatus = job.getStatus(); + if (job.getStatus() == PENDING) { + job.setStatus(RUNNING); + } + + JobResult result = job.getResult(); + if (jobStats.getTotalTasksCount() != null) { + result.setTotalCount(jobStats.getTotalTasksCount()); + } + + boolean publishEvent = false; + for (TaskResult taskResult : jobStats.getTaskResults()) { + if (!taskResult.getKey().equals(job.getConfiguration().getTasksKey())) { + log.debug("Ignoring task result {} with outdated key {}", taskResult, job.getConfiguration().getTasksKey()); + continue; + } + + result.processTaskResult(taskResult); + + if (result.getCancellationTs() > 0) { + if (!taskResult.isDiscarded() && System.currentTimeMillis() > result.getCancellationTs()) { + log.info("Got task result for cancelled job {}: {}, re-notifying processors about cancellation", jobId, taskResult); + // task processor forgot the task is cancelled + publishEvent = true; + } + } + } + + if (job.getStatus() == RUNNING) { + if (result.getTotalCount() != null && result.getCompletedCount() >= result.getTotalCount()) { + if (result.getCancellationTs() > 0) { + job.setStatus(CANCELLED); + } else if (result.getFailedCount() > 0) { + job.setStatus(FAILED); + publishEvent = true; + } else { + job.setStatus(COMPLETED); + publishEvent = true; + } + result.setFinishTs(System.currentTimeMillis()); + job.getConfiguration().setToReprocess(null); + } + } + + saveJob(tenantId, job, publishEvent, prevStatus); + } + + private Job saveJob(TenantId tenantId, Job job, boolean publishEvent, JobStatus prevStatus) { + ConstraintValidator.validateFields(job); + if (!Job.SUPPORTED_ENTITY_TYPES.contains(job.getEntityId().getEntityType())) { + throw new IllegalArgumentException("Unsupported entity type " + job.getEntityId().getEntityType()); + } + + job = jobDao.save(tenantId, job); + if (publishEvent) { + eventPublisher.publishEvent(SaveEntityEvent.builder() + .tenantId(tenantId) + .entityId(job.getId()) + .entity(job) + .build()); + } + log.info("[{}] Saved job: {}", tenantId, job); + if (prevStatus != null && job.getStatus() != prevStatus) { + log.info("[{}][{}][{}] New job status: {} -> {}", tenantId, job.getId(), job.getType(), prevStatus, job.getStatus()); + if (job.getStatus().isOneOf(CANCELLED, COMPLETED, FAILED) && prevStatus != QUEUED) { // if prev status is QUEUED - means there are already running jobs with this type, no need to check for waiting job + checkWaitingJobs(tenantId, job.getType()); + } + } + return job; + } + + private void checkWaitingJobs(TenantId tenantId, JobType jobType) { + Job queuedJob = jobDao.findOldestByTenantIdAndTypeAndStatusForUpdate(tenantId, jobType, QUEUED); + if (queuedJob == null) { + return; + } + queuedJob.setStatus(PENDING); + saveJob(tenantId, queuedJob, true, QUEUED); + } + + @Override + public PageData findJobsByFilter(TenantId tenantId, JobFilter filter, PageLink pageLink) { + PageData jobs = jobDao.findByTenantIdAndFilter(tenantId, filter, pageLink); + + Set entityIds = jobs.getData().stream() + .map(Job::getEntityId) + .collect(Collectors.toSet()); + Map entityInfos = entityService.fetchEntityInfos(tenantId, null, entityIds); + jobs.getData().forEach(job -> { + EntityInfo entityInfo = entityInfos.get(job.getEntityId()); + if (entityInfo != null) { + job.setEntityName(entityInfo.getName()); + } + }); + return jobs; + } + + @Override + public Job findLatestJobByKey(TenantId tenantId, String key) { + return jobDao.findLatestByTenantIdAndKey(tenantId, key); + } + + @Override + public void deleteJob(TenantId tenantId, JobId jobId) { + Job job = findJobById(tenantId, jobId); + if (!job.getStatus().isOneOf(CANCELLED, COMPLETED, FAILED)) { + throw new IllegalArgumentException("Job must be cancelled, completed or failed"); + } + jobDao.removeById(tenantId, jobId.getId()); + } + + @Override + public int deleteJobsByEntityId(TenantId tenantId, EntityId entityId) { // TODO: cancel all jobs for this entity + return jobDao.removeByEntityId(tenantId, entityId); + } + + private Job findForUpdate(TenantId tenantId, JobId jobId) { + return jobDao.findByIdForUpdate(tenantId, jobId); + } + + @Override + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findJobById(tenantId, (JobId) entityId)); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + jobDao.removeById(tenantId, id.getId()); + } + + @Override + public void deleteByTenantId(TenantId tenantId) { + jobDao.removeByTenantId(tenantId); + } + + @Override + public EntityType getEntityType() { + return EntityType.JOB; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/job/JobDao.java b/dao/src/main/java/org/thingsboard/server/dao/job/JobDao.java new file mode 100644 index 0000000000..498fc9f2de --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/job/JobDao.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.job; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobFilter; +import org.thingsboard.server.common.data.job.JobStatus; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.Dao; + +public interface JobDao extends Dao { + + PageData findByTenantIdAndFilter(TenantId tenantId, JobFilter filter, PageLink pageLink); + + Job findByIdForUpdate(TenantId tenantId, JobId jobId); + + Job findLatestByTenantIdAndKey(TenantId tenantId, String key); + + boolean existsByTenantAndKeyAndStatusOneOf(TenantId tenantId, String key, JobStatus... statuses); + + boolean existsByTenantIdAndTypeAndStatusOneOf(TenantId tenantId, JobType type, JobStatus... statuses); + + boolean existsByTenantIdAndEntityIdAndStatusOneOf(TenantId tenantId, EntityId entityId, JobStatus... statuses); + + Job findOldestByTenantIdAndTypeAndStatusForUpdate(TenantId tenantId, JobType type, JobStatus status); + + void removeByTenantId(TenantId tenantId); + + int removeByEntityId(TenantId tenantId, EntityId entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 148908d063..23324eccb5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -739,6 +739,18 @@ public class ModelConstants { public static final String CALCULATED_FIELD_LINK_ENTITY_ID = ENTITY_ID_COLUMN; public static final String CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID = "calculated_field_id"; + /** + * Tasks constants. + */ + public static final String JOB_TABLE_NAME = "job"; + public static final String JOB_TYPE_PROPERTY = "type"; + public static final String JOB_KEY_PROPERTY = "key"; + public static final String JOB_ENTITY_ID_PROPERTY = "entity_id"; + public static final String JOB_ENTITY_TYPE_PROPERTY = "entity_type"; + public static final String JOB_STATUS_PROPERTY = "status"; + public static final String JOB_CONFIGURATION_PROPERTY = "configuration"; + public static final String JOB_RESULT_PROPERTY = "result"; + protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/JobEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/JobEntity.java new file mode 100644 index 0000000000..16d7d33edc --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/JobEntity.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobConfiguration; +import org.thingsboard.server.common.data.job.JobResult; +import org.thingsboard.server.common.data.job.JobStatus; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.util.mapping.JsonConverter; + +import java.util.UUID; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@Entity +@Table(name = ModelConstants.JOB_TABLE_NAME) +public class JobEntity extends BaseSqlEntity { + + @Column(name = ModelConstants.TENANT_ID_PROPERTY, nullable = false) + private UUID tenantId; + + @Enumerated(EnumType.STRING) + @Column(name = ModelConstants.JOB_TYPE_PROPERTY, nullable = false) + private JobType type; + + @Column(name = ModelConstants.JOB_KEY_PROPERTY, nullable = false) + private String key; + + @Column(name = ModelConstants.JOB_ENTITY_ID_PROPERTY, nullable = false) + private UUID entityId; + + @Enumerated(EnumType.STRING) + @Column(name = ModelConstants.JOB_ENTITY_TYPE_PROPERTY, nullable = false) + private EntityType entityType; + + @Enumerated(EnumType.STRING) + @Column(name = ModelConstants.JOB_STATUS_PROPERTY, nullable = false) + private JobStatus status; + + @Convert(converter = JsonConverter.class) + @Column(name = ModelConstants.JOB_CONFIGURATION_PROPERTY, nullable = false) + private JsonNode configuration; + + @Convert(converter = JsonConverter.class) + @Column(name = ModelConstants.JOB_RESULT_PROPERTY) + private JsonNode result; + + public JobEntity(Job job) { + super(job); + this.tenantId = getTenantUuid(job.getTenantId()); + this.type = job.getType(); + this.key = job.getKey(); + this.entityId = job.getEntityId().getId(); + this.entityType = job.getEntityId().getEntityType(); + this.status = job.getStatus(); + this.configuration = toJson(job.getConfiguration()); + this.result = toJson(job.getResult()); + } + + @Override + public Job toData() { + Job job = new Job(); + job.setId(new JobId(id)); + job.setCreatedTime(createdTime); + job.setTenantId(getTenantId(tenantId)); + job.setType(type); + job.setKey(key); + job.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + job.setStatus(status); + job.setConfiguration(fromJson(configuration, JobConfiguration.class)); + job.setResult(fromJson(result, JobResult.class)); + return job; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/job/JobRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/job/JobRepository.java new file mode 100644 index 0000000000..df391be1b1 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/job/JobRepository.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.job; + +import org.springframework.data.domain.Limit; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.job.JobStatus; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.dao.model.sql.JobEntity; + +import java.util.List; +import java.util.UUID; + +@Repository +public interface JobRepository extends JpaRepository { + + @Query("SELECT j FROM JobEntity j WHERE j.tenantId = :tenantId " + + "AND (:types IS NULL OR j.type IN (:types)) " + + "AND (:statuses IS NULL OR j.status IN (:statuses)) " + + "AND (:entities IS NULL OR j.entityId IN :entities) " + + "AND (:startTime <= 0 OR j.createdTime >= :startTime) " + + "AND (:endTime <= 0 OR j.createdTime <= :endTime) " + + "AND (:searchText IS NULL OR ilike(j.key, concat('%', :searchText, '%')) = true)") + Page findByTenantIdAndTypesAndStatusesAndEntitiesAndTimeAndSearchText(@Param("tenantId") UUID tenantId, + @Param("types") List types, + @Param("statuses") List statuses, + @Param("entities") List entities, + @Param("startTime") long startTime, + @Param("endTime") long endTime, + @Param("searchText") String searchText, + Pageable pageable); + + @Query(value = "SELECT * FROM job j WHERE j.id = :id FOR UPDATE", nativeQuery = true) + JobEntity findByIdForUpdate(UUID id); + + @Query("SELECT j FROM JobEntity j WHERE j.tenantId = :tenantId AND j.key = :key " + + "ORDER BY j.createdTime DESC") + JobEntity findLatestByTenantIdAndKey(@Param("tenantId") UUID tenantId, @Param("key") String key, Limit limit); + + boolean existsByTenantIdAndKeyAndStatusIn(UUID tenantId, String key, List statuses); + + boolean existsByTenantIdAndTypeAndStatusIn(UUID tenantId, JobType type, List statuses); + + boolean existsByTenantIdAndEntityIdAndStatusIn(UUID tenantId, UUID entityId, List statuses); + + @Query(value = "SELECT * FROM job j WHERE j.tenant_id = :tenantId AND j.type = :type " + + "AND j.status = :status ORDER BY j.created_time ASC, j.id ASC LIMIT 1 FOR UPDATE", nativeQuery = true) + JobEntity findOldestByTenantIdAndTypeAndStatusForUpdate(UUID tenantId, String type, String status); + + @Transactional + @Modifying + @Query("DELETE FROM JobEntity j WHERE j.tenantId = :tenantId") + void deleteByTenantId(UUID tenantId); + + @Transactional + @Modifying + @Query("DELETE FROM JobEntity j WHERE j.entityId = :entityId") + int deleteByEntityId(UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/job/JpaJobDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/job/JpaJobDao.java new file mode 100644 index 0000000000..40d5177ad6 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/job/JpaJobDao.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.job; + +import com.google.common.base.Strings; +import lombok.RequiredArgsConstructor; +import org.springframework.data.domain.Limit; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.Job; +import org.thingsboard.server.common.data.job.JobFilter; +import org.thingsboard.server.common.data.job.JobStatus; +import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.job.JobDao; +import org.thingsboard.server.dao.model.sql.JobEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.Arrays; +import java.util.UUID; + +@Component +@SqlDao +@RequiredArgsConstructor +public class JpaJobDao extends JpaAbstractDao implements JobDao { + + private final JobRepository jobRepository; + + @Override + public PageData findByTenantIdAndFilter(TenantId tenantId, JobFilter filter, PageLink pageLink) { + return DaoUtil.toPageData(jobRepository.findByTenantIdAndTypesAndStatusesAndEntitiesAndTimeAndSearchText(tenantId.getId(), + CollectionsUtil.isEmpty(filter.getTypes()) ? null : filter.getTypes(), + CollectionsUtil.isEmpty(filter.getStatuses()) ? null : filter.getStatuses(), + CollectionsUtil.isEmpty(filter.getEntities()) ? null : filter.getEntities(), + filter.getStartTime() != null ? filter.getStartTime() : 0, + filter.getEndTime() != null ? filter.getEndTime() : 0, + Strings.emptyToNull(pageLink.getTextSearch()), DaoUtil.toPageable(pageLink))); + } + + @Override + public Job findByIdForUpdate(TenantId tenantId, JobId jobId) { + return DaoUtil.getData(jobRepository.findByIdForUpdate(jobId.getId())); + } + + @Override + public Job findLatestByTenantIdAndKey(TenantId tenantId, String key) { + return DaoUtil.getData(jobRepository.findLatestByTenantIdAndKey(tenantId.getId(), key, Limit.of(1))); + } + + @Override + public boolean existsByTenantAndKeyAndStatusOneOf(TenantId tenantId, String key, JobStatus... statuses) { + return jobRepository.existsByTenantIdAndKeyAndStatusIn(tenantId.getId(), key, Arrays.stream(statuses).toList()); + } + + @Override + public boolean existsByTenantIdAndTypeAndStatusOneOf(TenantId tenantId, JobType type, JobStatus... statuses) { + return jobRepository.existsByTenantIdAndTypeAndStatusIn(tenantId.getId(), type, Arrays.stream(statuses).toList()); + } + + @Override + public boolean existsByTenantIdAndEntityIdAndStatusOneOf(TenantId tenantId, EntityId entityId, JobStatus... statuses) { + return jobRepository.existsByTenantIdAndEntityIdAndStatusIn(tenantId.getId(), entityId.getId(), Arrays.stream(statuses).toList()); + } + + @Override + public Job findOldestByTenantIdAndTypeAndStatusForUpdate(TenantId tenantId, JobType type, JobStatus status) { + return DaoUtil.getData(jobRepository.findOldestByTenantIdAndTypeAndStatusForUpdate(tenantId.getId(), type.name(), status.name())); + } + + @Override + public void removeByTenantId(TenantId tenantId) { + jobRepository.deleteByTenantId(tenantId.getId()); + } + + @Override + public int removeByEntityId(TenantId tenantId, EntityId entityId) { + return jobRepository.deleteByEntityId(entityId.getId()); + } + + @Override + public EntityType getEntityType() { + return EntityType.JOB; + } + + @Override + protected Class getEntityClass() { + return JobEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return jobRepository; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 8c40ca3e14..02bfe5defb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -176,7 +176,7 @@ public class TenantServiceImpl extends AbstractCachedEntityService executor = LazyInitializer.builder() + .setInitializer(() -> ThingsBoardExecutors.newWorkStealingPool(10, getClass())) + .get(); protected final RestTemplate restTemplate; protected final RestTemplate loginRestTemplate; protected final String baseURL; @@ -224,22 +228,30 @@ public class RestClient implements Closeable { } public RestClient(RestTemplate restTemplate, String baseURL) { + this(restTemplate, baseURL, null); + } + + public RestClient(RestTemplate restTemplate, String baseURL, String accessToken) { this.restTemplate = restTemplate; this.loginRestTemplate = new RestTemplate(restTemplate.getRequestFactory()); this.baseURL = baseURL; this.restTemplate.getInterceptors().add((request, bytes, execution) -> { HttpRequest wrapper = new HttpRequestWrapper(request); - long calculatedTs = System.currentTimeMillis() + clientServerTimeDiff + AVG_REQUEST_TIMEOUT; - if (calculatedTs > mainTokenExpTs) { - synchronized (RestClient.this) { - if (calculatedTs > mainTokenExpTs) { - if (calculatedTs < refreshTokenExpTs) { - refreshToken(); - } else { - doLogin(); + if (accessToken == null) { + long calculatedTs = System.currentTimeMillis() + clientServerTimeDiff + AVG_REQUEST_TIMEOUT; + if (calculatedTs > mainTokenExpTs) { + synchronized (RestClient.this) { + if (calculatedTs > mainTokenExpTs) { + if (calculatedTs < refreshTokenExpTs) { + refreshToken(); + } else { + doLogin(); + } } } } + } else { + mainToken = accessToken; } wrapper.getHeaders().set(JWT_TOKEN_HEADER_PARAM, "Bearer " + mainToken); return execution.execute(wrapper, bytes); @@ -2404,7 +2416,7 @@ public class RestClient implements Closeable { } public Future> getAttributeKvEntriesAsync(EntityId entityId, List keys) { - return service.submit(() -> getAttributeKvEntries(entityId, keys)); + return getExecutor().submit(() -> getAttributeKvEntries(entityId, keys)); } public List getAttributesByScope(EntityId entityId, String scope, List keys) { @@ -2977,7 +2989,7 @@ public class RestClient implements Closeable { addWidgetInfoFiltersToParams(tenantOnly, fullSearch, deprecatedFilter, widgetTypeList, params); return restTemplate.exchange( baseURL + "/api/widgetTypes?" + getUrlParams(pageLink) + - getWidgetTypeInfoPageRequestUrlParams(tenantOnly, fullSearch, deprecatedFilter, widgetTypeList), + getWidgetTypeInfoPageRequestUrlParams(tenantOnly, fullSearch, deprecatedFilter, widgetTypeList), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>() { @@ -3065,7 +3077,7 @@ public class RestClient implements Closeable { addWidgetInfoFiltersToParams(tenantOnly, fullSearch, deprecatedFilter, widgetTypeList, params); return restTemplate.exchange( baseURL + "/api/widgetTypesInfos?widgetsBundleId={widgetsBundleId}&" + getUrlParams(pageLink) + - getWidgetTypeInfoPageRequestUrlParams(tenantOnly, fullSearch, deprecatedFilter, widgetTypeList), + getWidgetTypeInfoPageRequestUrlParams(tenantOnly, fullSearch, deprecatedFilter, widgetTypeList), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>() { @@ -4191,7 +4203,14 @@ public class RestClient implements Closeable { @Override public void close() { - service.shutdown(); + if (executor.isInitialized()) { + getExecutor().shutdown(); + } + } + + @SneakyThrows + private ExecutorService getExecutor() { + return executor.get(); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/JobManager.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/JobManager.java new file mode 100644 index 0000000000..ed8931f88e --- /dev/null +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/JobManager.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.JobId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.job.Job; + +public interface JobManager { + + ListenableFuture submitJob(Job job); // TODO: rate limits + + void cancelJob(TenantId tenantId, JobId jobId); + + void reprocessJob(TenantId tenantId, JobId jobId); + + void onJobUpdate(Job job); + +} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 7989b8f9ce..16f2936964 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -62,6 +62,7 @@ import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.job.JobService; import org.thingsboard.server.dao.mobile.MobileAppBundleService; import org.thingsboard.server.dao.mobile.MobileAppService; import org.thingsboard.server.dao.nosql.CassandraStatementTask; @@ -362,6 +363,10 @@ public interface TbContext { RuleEngineCalculatedFieldQueueService getCalculatedFieldQueueService(); + JobService getJobService(); + + JobManager getJobManager(); + boolean isExternalNodeForceAck(); /** diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java index f12a856567..93fad4c0e7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.DomainId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.JobId; import org.thingsboard.server.common.data.id.MobileAppBundleId; import org.thingsboard.server.common.data.id.MobileAppId; import org.thingsboard.server.common.data.id.NotificationRequestId; @@ -175,6 +176,9 @@ public class TenantIdLoader { tenantEntity = null; } break; + case JOB: + tenantEntity = ctx.getJobService().findJobById(ctxTenantId, new JobId(id)); + break; default: throw new RuntimeException("Unexpected entity type: " + entityId.getEntityType()); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index 38417c3922..4cbc091bdc 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -54,6 +54,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.job.Job; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.mobile.bundle.MobileAppBundle; import org.thingsboard.server.common.data.notification.NotificationRequest; @@ -88,6 +89,7 @@ import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.job.JobService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -160,6 +162,8 @@ public class TenantIdLoaderTest { private MobileAppBundleService mobileAppBundleService; @Mock private CalculatedFieldService calculatedFieldService; + @Mock + private JobService jobService; private TenantId tenantId; private TenantProfileId tenantProfileId; @@ -419,6 +423,12 @@ public class TenantIdLoaderTest { when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); doReturn(calculatedFieldLink).when(calculatedFieldService).findCalculatedFieldLinkById(eq(tenantId), any()); break; + case JOB: + Job job = new Job(); + job.setTenantId(tenantId); + when(ctx.getJobService()).thenReturn(jobService); + doReturn(job).when(jobService).findJobById(eq(tenantId), any()); + break; default: throw new RuntimeException("Unexpected originator EntityType " + entityType); } diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index 4be96f5fdd..eeb835e63f 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -21,7 +21,8 @@ import { Component, ElementRef, EventEmitter, - Input, NgZone, + Input, + NgZone, OnChanges, OnDestroy, OnInit, @@ -59,7 +60,7 @@ import { EntityTypeTranslation } from '@shared/models/entity-type.models'; import { DialogService } from '@core/services/dialog.service'; import { AddEntityDialogComponent } from './add-entity-dialog.component'; import { AddEntityDialogData, EntityAction } from '@home/models/entity/entity-component.models'; -import { calculateIntervalStartEndTime, HistoryWindowType, Timewindow } from '@shared/models/time/time.models'; +import { getTimePageLinkInterval, Timewindow } from '@shared/models/time/time.models'; import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; import { TbAnchorComponent } from '@shared/components/tb-anchor.component'; import { isDefined, isEqual, isNotEmptyStr, isUndefined } from '@core/utils'; @@ -259,7 +260,7 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa if (this.entitiesTableConfig.useTimePageLink) { this.timewindow = this.entitiesTableConfig.defaultTimewindowInterval; - const interval = this.getTimePageLinkInterval(); + const interval = getTimePageLinkInterval(this.timewindow); this.pageLink = new TimePageLink(10, 0, null, sortOrder, interval.startTime, interval.endTime); } else { @@ -424,7 +425,7 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa } if (this.entitiesTableConfig.useTimePageLink) { const timePageLink = this.pageLink as TimePageLink; - const interval = this.getTimePageLinkInterval(); + const interval = getTimePageLinkInterval(this.timewindow); timePageLink.startTime = interval.startTime; timePageLink.endTime = interval.endTime; } @@ -434,31 +435,6 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa } } - private getTimePageLinkInterval(): {startTime?: number; endTime?: number} { - const interval: {startTime?: number; endTime?: number} = {}; - switch (this.timewindow.history.historyType) { - case HistoryWindowType.LAST_INTERVAL: - const currentTime = Date.now(); - interval.startTime = currentTime - this.timewindow.history.timewindowMs; - interval.endTime = currentTime; - break; - case HistoryWindowType.FIXED: - interval.startTime = this.timewindow.history.fixedTimewindow.startTimeMs; - interval.endTime = this.timewindow.history.fixedTimewindow.endTimeMs; - break; - case HistoryWindowType.INTERVAL: - const startEndTime = calculateIntervalStartEndTime(this.timewindow.history.quickInterval); - interval.startTime = startEndTime[0]; - interval.endTime = startEndTime[1]; - break; - case HistoryWindowType.FOR_ALL_TIME: - interval.startTime = null; - interval.endTime = null; - break; - } - return interval; - } - private dataLoaded(col?: number, row?: number) { if (isFinite(col) && isFinite(row)) { this.clearCellCache(col, row); diff --git a/ui-ngx/src/app/shared/components/entity/entity-list-select.component.html b/ui-ngx/src/app/shared/components/entity/entity-list-select.component.html index 1701e8d754..39c2da9c2f 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-list-select.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-list-select.component.html @@ -17,16 +17,20 @@ -->
{ }; + private propagateChange = (_v: any) => { }; - constructor(private store: Store, - private entityService: EntityService, - public translate: TranslateService, + constructor(private entityService: EntityService, private fb: UntypedFormBuilder, private destroyRef: DestroyRef) { @@ -96,7 +94,7 @@ export class EntityListSelectComponent implements ControlValueAccessor, OnInit, this.propagateChange = fn; } - registerOnTouched(fn: any): void { + registerOnTouched(_fn: any): void { } ngOnInit() { @@ -114,9 +112,9 @@ export class EntityListSelectComponent implements ControlValueAccessor, OnInit, this.updateView(this.modelValue.entityType, values); } ); - } - - ngAfterViewInit(): void { + if (isDefinedAndNotNull(this.predefinedEntityType)) { + this.defaultEntityType = this.predefinedEntityType; + } } setDisabledState(isDisabled: boolean): void { @@ -145,7 +143,7 @@ export class EntityListSelectComponent implements ControlValueAccessor, OnInit, this.entityListSelectFormGroup.get('entityIds').patchValue([...this.modelValue.ids], {emitEvent: true}); } - updateView(entityType: EntityType | AliasEntityType | null, entityIds: Array | null) { + private updateView(entityType: EntityType | AliasEntityType | null, entityIds: Array | null) { if (this.modelValue.entityType !== entityType || !this.compareIds(this.modelValue.ids, entityIds)) { this.modelValue = { @@ -156,7 +154,7 @@ export class EntityListSelectComponent implements ControlValueAccessor, OnInit, } } - compareIds(ids1: Array | null, ids2: Array | null): boolean { + private compareIds(ids1: Array | null, ids2: Array | null): boolean { if (ids1 !== null && ids2 !== null) { return JSON.stringify(ids1) === JSON.stringify(ids2); } else { @@ -164,7 +162,7 @@ export class EntityListSelectComponent implements ControlValueAccessor, OnInit, } } - toEntityIds(modelValue: EntityListSelectModel): Array { + private toEntityIds(modelValue: EntityListSelectModel): Array { if (modelValue !== null && modelValue.entityType && modelValue.ids && modelValue.ids.length > 0) { const entityType = modelValue.entityType; return modelValue.ids.map(id => ({entityType, id})); diff --git a/ui-ngx/src/app/shared/components/entity/entity-list.component.html b/ui-ngx/src/app/shared/components/entity/entity-list.component.html index fc0cfa9586..e1b951b40b 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-list.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-list.component.html @@ -15,8 +15,13 @@ limitations under the License. --> - - {{ labelText }} + + {{ labelText }} {{ labelText }} - {{ translate.get('entity.no-entities-matching', {entity: searchText}) | async }} + {{ 'entity.no-entities-matching' | translate: {entity: searchText} }}
- + {{ hint }} - + {{ requiredText }}
diff --git a/ui-ngx/src/app/shared/components/entity/entity-list.component.ts b/ui-ngx/src/app/shared/components/entity/entity-list.component.ts index 5cd19156a5..552c4f1f71 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-list.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-list.component.ts @@ -14,17 +14,7 @@ /// limitations under the License. /// -import { - AfterViewInit, - Component, - ElementRef, - forwardRef, - Input, - OnChanges, - OnInit, - SimpleChanges, - ViewChild -} from '@angular/core'; +import { Component, ElementRef, forwardRef, Input, OnChanges, OnInit, SimpleChanges, ViewChild } from '@angular/core'; import { ControlValueAccessor, NG_VALIDATORS, @@ -65,7 +55,7 @@ import { isArray } from 'lodash'; } ] }) -export class EntityListComponent implements ControlValueAccessor, OnInit, AfterViewInit, OnChanges { +export class EntityListComponent implements ControlValueAccessor, OnInit, OnChanges { entityListFormGroup: UntypedFormGroup; @@ -115,6 +105,10 @@ export class EntityListComponent implements ControlValueAccessor, OnInit, AfterV @coerceBoolean() syncIdsWithDB = false; + @Input() + @coerceBoolean() + inlineField: boolean; + @ViewChild('entityInput') entityInput: ElementRef; @ViewChild('entityAutocomplete') matAutocomplete: MatAutocomplete; @ViewChild('chipList', {static: true}) chipList: MatChipGrid; @@ -126,9 +120,9 @@ export class EntityListComponent implements ControlValueAccessor, OnInit, AfterV private dirty = false; - private propagateChange = (v: any) => { }; + private propagateChange = (_v: any) => { }; - constructor(public translate: TranslateService, + constructor(private translate: TranslateService, private entityService: EntityService, private fb: UntypedFormBuilder) { this.entityListFormGroup = this.fb.group({ @@ -146,7 +140,7 @@ export class EntityListComponent implements ControlValueAccessor, OnInit, AfterV this.propagateChange = fn; } - registerOnTouched(fn: any): void { + registerOnTouched(_fn: any): void { } ngOnInit() { @@ -178,9 +172,6 @@ export class EntityListComponent implements ControlValueAccessor, OnInit, AfterV } } - ngAfterViewInit(): void { - } - setDisabledState(isDisabled: boolean): void { this.disabled = isDisabled; if (isDisabled) { diff --git a/ui-ngx/src/app/shared/components/entity/entity-type-select.component.html b/ui-ngx/src/app/shared/components/entity/entity-type-select.component.html index b21734cfdd..6506d233b0 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-type-select.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-type-select.component.html @@ -15,14 +15,16 @@ limitations under the License. --> - - {{ 'entity.type' | translate }} + + {{ label }} {{ displayEntityTypeFn(type) }} - + {{ 'entity.type-required' | translate }} diff --git a/ui-ngx/src/app/shared/components/entity/entity-type-select.component.ts b/ui-ngx/src/app/shared/components/entity/entity-type-select.component.ts index ce599a0c08..f8a4f49a82 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-type-select.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-type-select.component.ts @@ -52,6 +52,9 @@ export class EntityTypeSelectComponent implements ControlValueAccessor, OnInit, @coerceBoolean() showLabel: boolean; + @Input() + label = this.translate.instant('entity.type'); + @Input() @coerceBoolean() required: boolean; @@ -65,12 +68,16 @@ export class EntityTypeSelectComponent implements ControlValueAccessor, OnInit, @Input() appearance: MatFormFieldAppearance = 'fill'; + @Input() + @coerceBoolean() + inlineField: boolean; + entityTypes: Array; - private propagateChange = (v: any) => { }; + private propagateChange = (_v: any) => { }; constructor(private entityService: EntityService, - public translate: TranslateService, + private translate: TranslateService, private fb: UntypedFormBuilder, private destroyRef: DestroyRef) { this.entityTypeFormGroup = this.fb.group({ @@ -82,7 +89,7 @@ export class EntityTypeSelectComponent implements ControlValueAccessor, OnInit, this.propagateChange = fn; } - registerOnTouched(fn: any): void { + registerOnTouched(_fn: any): void { } ngOnInit() { @@ -97,7 +104,7 @@ export class EntityTypeSelectComponent implements ControlValueAccessor, OnInit, takeUntilDestroyed(this.destroyRef) ).subscribe( (value) => { - let modelValue; + let modelValue: EntityType | AliasEntityType; if (!value || value === '') { modelValue = null; } else { diff --git a/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html b/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html index 042d94087c..fbb3c1d2f6 100644 --- a/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html +++ b/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html @@ -27,7 +27,7 @@
-
+
- -
+ +
- - -
{{ computedTimewindowStyle.icon }}
+ diff --git a/ui-ngx/src/app/shared/components/time/timewindow.component.ts b/ui-ngx/src/app/shared/components/time/timewindow.component.ts index ea3f49cf9f..a52e3eb091 100644 --- a/ui-ngx/src/app/shared/components/time/timewindow.component.ts +++ b/ui-ngx/src/app/shared/components/time/timewindow.component.ts @@ -17,6 +17,7 @@ import { ChangeDetectorRef, Component, + DestroyRef, ElementRef, forwardRef, HostBinding, @@ -26,6 +27,7 @@ import { OnInit, SimpleChanges, StaticProvider, + ViewChild, ViewContainerRef } from '@angular/core'; import { ControlValueAccessor, NG_VALUE_ACCESSOR } from '@angular/forms'; @@ -63,6 +65,7 @@ import { } from '@shared/models/widget-settings.models'; import { DEFAULT_OVERLAY_POSITIONS } from '@shared/models/overlay.models'; import { fromEvent } from 'rxjs'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; // @dynamic @Component({ @@ -79,6 +82,8 @@ import { fromEvent } from 'rxjs'; }) export class TimewindowComponent implements ControlValueAccessor, OnInit, OnChanges { + @ViewChild('panelContainer', { read: ViewContainerRef, static: true }) panelContainer: ViewContainerRef; + historyOnlyValue = false; @Input() @@ -180,6 +185,10 @@ export class TimewindowComponent implements ControlValueAccessor, OnInit, OnChan @coerceBoolean() disabled: boolean; + @Input() + @coerceBoolean() + panelMode = true; + innerValue: Timewindow; timewindowDisabled: boolean; @@ -197,7 +206,8 @@ export class TimewindowComponent implements ControlValueAccessor, OnInit, OnChan private datePipe: DatePipe, private cd: ChangeDetectorRef, private nativeElement: ElementRef, - public viewContainerRef: ViewContainerRef) { + private viewContainerRef: ViewContainerRef, + private destroyRef: DestroyRef) { } ngOnInit() { @@ -249,7 +259,8 @@ export class TimewindowComponent implements ControlValueAccessor, OnInit, OnChan quickIntervalOnly: this.quickIntervalOnly, aggregation: this.aggregation, timezone: this.timezone, - isEdit: this.isEdit + isEdit: this.isEdit, + panelMode: this.panelMode, } as TimewindowPanelData }, { @@ -317,6 +328,9 @@ export class TimewindowComponent implements ControlValueAccessor, OnInit, OnChan } else { this.updateDisplayValue(); } + if (!this.panelMode) { + this.createPanel(); + } } notifyChanged() { @@ -328,6 +342,9 @@ export class TimewindowComponent implements ControlValueAccessor, OnInit, OnChan } updateDisplayValue() { + if (!this.panelMode) { + return + } if (this.innerValue.selectedTab === TimewindowType.REALTIME && !this.historyOnly) { this.innerValue.displayValue = this.displayTypePrefix ? (this.translate.instant('timewindow.realtime') + ' - ') : ''; if (this.innerValue.realtime.realtimeType === RealtimeWindowType.INTERVAL) { @@ -373,4 +390,29 @@ export class TimewindowComponent implements ControlValueAccessor, OnInit, OnChan ))); } + private createPanel() { + this.panelContainer.clear(); + const panelData = { + timewindow: deepClone(this.innerValue), + historyOnly: this.historyOnly, + forAllTimeEnabled: this.forAllTimeEnabled, + quickIntervalOnly: this.quickIntervalOnly, + aggregation: this.aggregation, + timezone: this.timezone, + isEdit: this.isEdit, + panelMode: this.panelMode, + } + const injector = Injector.create({ + providers: [{ provide: TIMEWINDOW_PANEL_DATA, useValue: panelData }], + parent: this.viewContainerRef.injector + }); + const componentRef = this.panelContainer.createComponent(TimewindowPanelComponent, {index: 0, injector}); + componentRef.instance.changeTimewindow.pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe(value => { + this.innerValue = value; + this.timewindowDisabled = this.isTimewindowDisabled(); + this.notifyChanged(); + }) + } } diff --git a/ui-ngx/src/app/shared/models/time/time.models.ts b/ui-ngx/src/app/shared/models/time/time.models.ts index c63bd8da52..acb62d9d2a 100644 --- a/ui-ngx/src/app/shared/models/time/time.models.ts +++ b/ui-ngx/src/app/shared/models/time/time.models.ts @@ -1406,3 +1406,28 @@ export const calculateInterval = (startTime: number, endTime: number, export const getCurrentTimeForComparison = (timeForComparison: moment_.unitOfTime.DurationConstructor, tz?: string): moment_.Moment => getCurrentTime(tz).subtract(1, timeForComparison); + +export const getTimePageLinkInterval = (timewindow: Timewindow): {startTime?: number; endTime?: number} => { + const interval: {startTime?: number; endTime?: number} = {}; + switch (timewindow.history.historyType) { + case HistoryWindowType.LAST_INTERVAL: + const currentTime = Date.now(); + interval.startTime = currentTime - timewindow.history.timewindowMs; + interval.endTime = currentTime; + break; + case HistoryWindowType.FIXED: + interval.startTime = timewindow.history.fixedTimewindow.startTimeMs; + interval.endTime = timewindow.history.fixedTimewindow.endTimeMs; + break; + case HistoryWindowType.INTERVAL: + const startEndTime = calculateIntervalStartEndTime(timewindow.history.quickInterval); + interval.startTime = startEndTime[0]; + interval.endTime = startEndTime[1]; + break; + case HistoryWindowType.FOR_ALL_TIME: + interval.startTime = null; + interval.endTime = null; + break; + } + return interval; +}