38 changed files with 1498 additions and 4 deletions
@ -0,0 +1,83 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.job; |
||||
|
|
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import org.springframework.stereotype.Component; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.ProfileEntityIdInfo; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedField; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.job.CfReprocessingJobConfiguration; |
||||
|
import org.thingsboard.server.common.data.job.CfReprocessingTask; |
||||
|
import org.thingsboard.server.common.data.job.Job; |
||||
|
import org.thingsboard.server.common.data.job.JobType; |
||||
|
import org.thingsboard.server.common.data.job.Task; |
||||
|
import org.thingsboard.server.common.data.page.PageDataIterable; |
||||
|
import org.thingsboard.server.dao.asset.AssetService; |
||||
|
import org.thingsboard.server.dao.device.DeviceService; |
||||
|
|
||||
|
import java.util.function.Consumer; |
||||
|
|
||||
|
@Component |
||||
|
@RequiredArgsConstructor |
||||
|
public class CfReprocessingJobProcessor extends JobProcessor { |
||||
|
|
||||
|
private final DeviceService deviceService; |
||||
|
private final AssetService assetService; |
||||
|
|
||||
|
@Override |
||||
|
public void process(Job job, Consumer<Task> taskConsumer) { |
||||
|
CfReprocessingJobConfiguration configuration = job.getConfiguration(); |
||||
|
|
||||
|
CalculatedField calculatedField = configuration.getCalculatedField(); |
||||
|
EntityId entityId = calculatedField.getEntityId(); |
||||
|
|
||||
|
if (entityId.getEntityType().isOneOf(EntityType.DEVICE, EntityType.ASSET)) { |
||||
|
taskConsumer.accept(createTask(job, configuration, entityId)); |
||||
|
} else { |
||||
|
PageDataIterable<ProfileEntityIdInfo> entities; |
||||
|
if (entityId.getEntityType() == EntityType.DEVICE_PROFILE) { |
||||
|
entities = new PageDataIterable<>(pageLink -> deviceService.findProfileEntityIdInfosByTenantId(job.getTenantId(), pageLink), 512); |
||||
|
} else if (entityId.getEntityType() == EntityType.ASSET_PROFILE) { |
||||
|
entities = new PageDataIterable<>(pageLink -> assetService.findProfileEntityIdInfosByTenantId(job.getTenantId(), pageLink), 512); |
||||
|
} else { |
||||
|
throw new IllegalArgumentException("Unsupported CF entity type " + entityId.getEntityType()); |
||||
|
} |
||||
|
entities.forEach(device -> { |
||||
|
taskConsumer.accept(createTask(job, configuration, device.getEntityId())); |
||||
|
}); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private Task createTask(Job job, CfReprocessingJobConfiguration configuration, EntityId entityId) { |
||||
|
return CfReprocessingTask.builder() |
||||
|
.tenantId(job.getTenantId()) |
||||
|
.jobId(job.getId()) |
||||
|
.key(entityId.getEntityType().getNormalName() + " " + entityId.getId()) |
||||
|
.calculatedField(configuration.getCalculatedField()) |
||||
|
.entityId(entityId) |
||||
|
.startTs(configuration.getStartTs()) |
||||
|
.endTs(configuration.getEndTs()) |
||||
|
.build(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public JobType getType() { |
||||
|
return JobType.CF_REPROCESSING; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,136 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.job; |
||||
|
|
||||
|
import jakarta.annotation.PreDestroy; |
||||
|
import lombok.SneakyThrows; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.stereotype.Component; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
||||
|
import org.thingsboard.server.common.data.id.JobId; |
||||
|
import org.thingsboard.server.common.data.job.Job; |
||||
|
import org.thingsboard.server.common.data.job.JobType; |
||||
|
import org.thingsboard.server.common.data.job.Task; |
||||
|
import org.thingsboard.server.common.data.job.TaskResult; |
||||
|
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
||||
|
import org.thingsboard.server.dao.task.JobService; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.TaskResultProto; |
||||
|
import org.thingsboard.server.queue.TbQueueCallback; |
||||
|
import org.thingsboard.server.queue.TbQueueConsumer; |
||||
|
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
||||
|
import org.thingsboard.server.queue.TbQueueProducer; |
||||
|
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
||||
|
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; |
||||
|
import org.thingsboard.server.queue.provider.TbCoreQueueFactory; |
||||
|
import org.thingsboard.server.queue.util.AfterStartUp; |
||||
|
import org.thingsboard.server.queue.util.TbCoreComponent; |
||||
|
|
||||
|
import java.util.Arrays; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.concurrent.ExecutorService; |
||||
|
import java.util.concurrent.Executors; |
||||
|
import java.util.function.Function; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
@TbCoreComponent |
||||
|
@Component |
||||
|
@Slf4j |
||||
|
public class DefaultJobManager implements JobManager { |
||||
|
|
||||
|
private final JobService jobService; |
||||
|
private final TbCoreQueueFactory queueFactory; |
||||
|
private final Map<JobType, JobProcessor> jobProcessors; |
||||
|
private final Map<JobType, TbQueueProducer<TbProtoQueueMsg<TaskProto>>> taskProducers; |
||||
|
private final QueueConsumerManager<TbProtoQueueMsg<TaskResultProto>> taskResultConsumer; |
||||
|
private final ExecutorService consumerExecutor; |
||||
|
|
||||
|
public DefaultJobManager(JobService jobService, TbCoreQueueFactory queueFactory, List<JobProcessor> jobProcessors) { |
||||
|
this.jobService = jobService; |
||||
|
this.queueFactory = queueFactory; |
||||
|
this.jobProcessors = jobProcessors.stream().collect(Collectors.toMap(JobProcessor::getType, Function.identity())); |
||||
|
this.taskProducers = Arrays.stream(JobType.values()).collect(Collectors.toMap(Function.identity(), queueFactory::createTaskProducer)); |
||||
|
this.consumerExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("task-result-consumer")); |
||||
|
this.taskResultConsumer = QueueConsumerManager.<TbProtoQueueMsg<TaskResultProto>>builder() // fixme: should be consumer per partition
|
||||
|
.name("tasks-results") |
||||
|
.msgPackProcessor(this::processResults) |
||||
|
.pollInterval(125) |
||||
|
.consumerCreator(queueFactory::createTaskResultConsumer) |
||||
|
.consumerExecutor(consumerExecutor) |
||||
|
.build(); |
||||
|
} |
||||
|
|
||||
|
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) |
||||
|
public void afterStartUp() { |
||||
|
taskResultConsumer.subscribe(); |
||||
|
taskResultConsumer.launch(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void submitJob(Job job) { |
||||
|
job = jobService.createJob(job.getTenantId(), job); |
||||
|
log.info("Submitting job: {}", job); |
||||
|
jobProcessors.get(job.getType()).process(job, this::submitTask); |
||||
|
} |
||||
|
|
||||
|
private void submitTask(Task task) { |
||||
|
log.info("Submitting task: {}", task); |
||||
|
TaskProto taskProto = TaskProto.newBuilder() |
||||
|
.setValue(JacksonUtil.toString(task)) |
||||
|
.build(); |
||||
|
|
||||
|
TbQueueProducer<TbProtoQueueMsg<TaskProto>> producer = taskProducers.get(task.getJobType()); |
||||
|
TbProtoQueueMsg<TaskProto> msg = new TbProtoQueueMsg<>(task.getTenantId().getId(), taskProto); // one job at a time for a given tenant
|
||||
|
producer.send(TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(), msg, new TbQueueCallback() { |
||||
|
@Override |
||||
|
public void onSuccess(TbQueueMsgMetadata metadata) { |
||||
|
log.trace("Submitted task: {}", task); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onFailure(Throwable t) { |
||||
|
log.warn("Failed to submit task: {}", task, t); |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
@SneakyThrows |
||||
|
private void processResults(List<TbProtoQueueMsg<TaskResultProto>> msgs, TbQueueConsumer<TbProtoQueueMsg<TaskResultProto>> consumer) { |
||||
|
Map<JobId, List<TaskResult>> results = msgs.stream() |
||||
|
.map(msg -> JacksonUtil.fromString(msg.getValue().getValue(), TaskResult.class)) |
||||
|
.collect(Collectors.groupingBy(TaskResult::getJobId)); |
||||
|
results.forEach((jobId, taskResults) -> { |
||||
|
try { |
||||
|
log.info("[{}] Processing task results: {}", jobId, taskResults); |
||||
|
jobService.reportTaskResults(jobId, taskResults); |
||||
|
} catch (Exception e) { |
||||
|
log.warn("Failed to report task results for job {}: {}", jobId, taskResults, e); |
||||
|
} |
||||
|
}); |
||||
|
consumer.commit(); |
||||
|
|
||||
|
Thread.sleep(5000); |
||||
|
} |
||||
|
|
||||
|
@PreDestroy |
||||
|
private void destroy() { |
||||
|
taskResultConsumer.stop(); |
||||
|
consumerExecutor.shutdownNow(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,24 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.job; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.job.Job; |
||||
|
|
||||
|
public interface JobManager { |
||||
|
|
||||
|
void submitJob(Job job); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,30 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.job; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.job.Task; |
||||
|
import org.thingsboard.server.common.data.job.Job; |
||||
|
import org.thingsboard.server.common.data.job.JobType; |
||||
|
|
||||
|
import java.util.function.Consumer; |
||||
|
|
||||
|
public abstract class JobProcessor { |
||||
|
|
||||
|
public abstract void process(Job job, Consumer<Task> taskConsumer); |
||||
|
|
||||
|
public abstract JobType getType(); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,57 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.job.task; |
||||
|
|
||||
|
import com.google.common.util.concurrent.SettableFuture; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import org.springframework.stereotype.Component; |
||||
|
import org.thingsboard.server.actors.calculatedField.CalculatedFieldReprocessingService; |
||||
|
import org.thingsboard.server.common.data.job.CfReprocessingTask; |
||||
|
import org.thingsboard.server.common.data.job.JobType; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.queue.task.TaskProcessor; |
||||
|
|
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
|
||||
|
@Component |
||||
|
@RequiredArgsConstructor |
||||
|
public class CfReprocessingTaskProcessor extends TaskProcessor<CfReprocessingTask> { |
||||
|
|
||||
|
private final CalculatedFieldReprocessingService cfReprocessingService; |
||||
|
|
||||
|
@Override |
||||
|
protected void process(CfReprocessingTask task) throws Exception { |
||||
|
SettableFuture<Void> future = SettableFuture.create(); |
||||
|
cfReprocessingService.reprocess(task, new TbCallback() { |
||||
|
@Override |
||||
|
public void onSuccess() { |
||||
|
future.set(null); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onFailure(Throwable t) { |
||||
|
future.setException(t); |
||||
|
} |
||||
|
}); |
||||
|
future.get(1, TimeUnit.MINUTES); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public JobType getJobType() { |
||||
|
return JobType.CF_REPROCESSING; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,35 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.dao.task; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.id.JobId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.job.TaskResult; |
||||
|
import org.thingsboard.server.common.data.page.PageData; |
||||
|
import org.thingsboard.server.common.data.page.PageLink; |
||||
|
import org.thingsboard.server.common.data.job.Job; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
public interface JobService { |
||||
|
|
||||
|
Job createJob(TenantId tenantId, Job job); |
||||
|
|
||||
|
void reportTaskResults(JobId jobId, List<TaskResult> results); |
||||
|
|
||||
|
PageData<Job> findJobsByTenantId(TenantId tenantId, PageLink pageLink); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,38 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.id; |
||||
|
|
||||
|
import com.fasterxml.jackson.annotation.JsonCreator; |
||||
|
import com.fasterxml.jackson.annotation.JsonProperty; |
||||
|
import io.swagger.v3.oas.annotations.media.Schema; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
|
||||
|
import java.util.UUID; |
||||
|
|
||||
|
public class JobId extends UUIDBased implements EntityId { |
||||
|
|
||||
|
@JsonCreator |
||||
|
public JobId(@JsonProperty("id") UUID id) { |
||||
|
super(id); |
||||
|
} |
||||
|
|
||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "TASK", allowableValues = "TASK") |
||||
|
@Override |
||||
|
public EntityType getEntityType() { |
||||
|
return EntityType.JOB; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,39 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.job; |
||||
|
|
||||
|
import lombok.AllArgsConstructor; |
||||
|
import lombok.Builder; |
||||
|
import lombok.Data; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedField; |
||||
|
|
||||
|
@Data |
||||
|
@AllArgsConstructor |
||||
|
@NoArgsConstructor |
||||
|
@Builder |
||||
|
public class CfReprocessingJobConfiguration implements JobConfiguration { |
||||
|
|
||||
|
private CalculatedField calculatedField; |
||||
|
private long startTs; |
||||
|
private long endTs; |
||||
|
|
||||
|
@Override |
||||
|
public JobType getType() { |
||||
|
return JobType.CF_REPROCESSING; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,25 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.job; |
||||
|
|
||||
|
public class CfReprocessingJobResult extends JobResult { |
||||
|
|
||||
|
@Override |
||||
|
public JobType getJobType() { |
||||
|
return JobType.CF_REPROCESSING; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,53 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.job; |
||||
|
|
||||
|
import lombok.AllArgsConstructor; |
||||
|
import lombok.Builder; |
||||
|
import lombok.Data; |
||||
|
import lombok.EqualsAndHashCode; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedField; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.JobId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
|
||||
|
@Data |
||||
|
@AllArgsConstructor |
||||
|
@NoArgsConstructor |
||||
|
@EqualsAndHashCode(callSuper = true) |
||||
|
public class CfReprocessingTask extends Task { |
||||
|
|
||||
|
private CalculatedField calculatedField; |
||||
|
private EntityId entityId; |
||||
|
private long startTs; |
||||
|
private long endTs; |
||||
|
|
||||
|
@Builder |
||||
|
public CfReprocessingTask(TenantId tenantId, JobId jobId, String key, CalculatedField calculatedField, EntityId entityId, long startTs, long endTs) { |
||||
|
super(tenantId, jobId, key); |
||||
|
this.calculatedField = calculatedField; |
||||
|
this.entityId = entityId; |
||||
|
this.startTs = startTs; |
||||
|
this.endTs = endTs; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public JobType getJobType() { |
||||
|
return JobType.CF_REPROCESSING; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,56 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.job; |
||||
|
|
||||
|
import lombok.Builder; |
||||
|
import lombok.Data; |
||||
|
import lombok.EqualsAndHashCode; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import org.thingsboard.server.common.data.BaseData; |
||||
|
import org.thingsboard.server.common.data.HasTenantId; |
||||
|
import org.thingsboard.server.common.data.id.JobId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
|
||||
|
@Data |
||||
|
@NoArgsConstructor |
||||
|
@EqualsAndHashCode(callSuper = true) |
||||
|
public class Job extends BaseData<JobId> implements HasTenantId { |
||||
|
|
||||
|
private TenantId tenantId; |
||||
|
private JobType type; |
||||
|
private String key; |
||||
|
private JobStatus status; |
||||
|
private JobConfiguration configuration; |
||||
|
private JobResult result; |
||||
|
|
||||
|
@Builder |
||||
|
public Job(TenantId tenantId, JobType type, String key, JobConfiguration configuration) { |
||||
|
this.tenantId = tenantId; |
||||
|
this.type = type; |
||||
|
this.key = key; |
||||
|
this.configuration = configuration; |
||||
|
this.status = JobStatus.PENDING; |
||||
|
this.result = switch (type) { |
||||
|
case CF_REPROCESSING -> new CfReprocessingJobResult(); |
||||
|
}; |
||||
|
} |
||||
|
|
||||
|
@SuppressWarnings("unchecked") |
||||
|
public <C extends JobConfiguration> C getConfiguration() { |
||||
|
return (C) configuration; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,32 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.job; |
||||
|
|
||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; |
||||
|
import com.fasterxml.jackson.annotation.JsonSubTypes; |
||||
|
import com.fasterxml.jackson.annotation.JsonSubTypes.Type; |
||||
|
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
||||
|
|
||||
|
@JsonIgnoreProperties(ignoreUnknown = true) |
||||
|
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") |
||||
|
@JsonSubTypes({ |
||||
|
@Type(name = "CF_REPROCESSING", value = CfReprocessingJobConfiguration.class), |
||||
|
}) |
||||
|
public interface JobConfiguration { |
||||
|
|
||||
|
JobType getType(); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,41 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.job; |
||||
|
|
||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; |
||||
|
import com.fasterxml.jackson.annotation.JsonSubTypes; |
||||
|
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
||||
|
import lombok.Data; |
||||
|
|
||||
|
import java.util.HashMap; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
@JsonIgnoreProperties(ignoreUnknown = true) |
||||
|
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "jobType") |
||||
|
@JsonSubTypes({ |
||||
|
@JsonSubTypes.Type(name = "CF_REPROCESSING", value = CfReprocessingJobResult.class), |
||||
|
}) |
||||
|
@Data |
||||
|
public abstract class JobResult { |
||||
|
|
||||
|
private int successfulCount; |
||||
|
private int failedCount; |
||||
|
private int totalCount; |
||||
|
private Map<String, String> failures = new HashMap<>(); |
||||
|
|
||||
|
public abstract JobType getJobType(); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,24 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.job; |
||||
|
|
||||
|
public enum JobStatus { |
||||
|
PENDING, |
||||
|
RUNNING, |
||||
|
COMPLETED, |
||||
|
FAILED, |
||||
|
CANCELLED |
||||
|
} |
||||
@ -0,0 +1,26 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.job; |
||||
|
|
||||
|
public enum JobType { |
||||
|
|
||||
|
CF_REPROCESSING; |
||||
|
|
||||
|
public String getTasksTopic() { |
||||
|
return "tasks." + name().toLowerCase(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,50 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.job; |
||||
|
|
||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; |
||||
|
import com.fasterxml.jackson.annotation.JsonSubTypes; |
||||
|
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.common.data.id.JobId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
|
||||
|
@Data |
||||
|
@JsonIgnoreProperties(ignoreUnknown = true) |
||||
|
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "jobType") |
||||
|
@JsonSubTypes({ |
||||
|
@JsonSubTypes.Type(name = "CF_REPROCESSING", value = CfReprocessingTask.class), |
||||
|
}) |
||||
|
public abstract class Task { |
||||
|
|
||||
|
private TenantId tenantId; |
||||
|
private JobId jobId; |
||||
|
private String key; |
||||
|
|
||||
|
public Task(TenantId tenantId, JobId jobId, String key) { |
||||
|
this.tenantId = tenantId; |
||||
|
this.jobId = jobId; |
||||
|
this.key = key; |
||||
|
} |
||||
|
|
||||
|
public Task() { |
||||
|
} |
||||
|
|
||||
|
private int attempt = 0; |
||||
|
|
||||
|
public abstract JobType getJobType(); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,45 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.common.data.job; |
||||
|
|
||||
|
import lombok.AllArgsConstructor; |
||||
|
import lombok.Builder; |
||||
|
import lombok.Data; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import org.thingsboard.server.common.data.id.JobId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
|
||||
|
@Data |
||||
|
@AllArgsConstructor |
||||
|
@NoArgsConstructor |
||||
|
@Builder |
||||
|
public class TaskResult { |
||||
|
|
||||
|
private TenantId tenantId; |
||||
|
private JobId jobId; |
||||
|
private boolean success; |
||||
|
private TaskFailure failure; |
||||
|
|
||||
|
@Data |
||||
|
@AllArgsConstructor |
||||
|
@NoArgsConstructor |
||||
|
@Builder |
||||
|
public static class TaskFailure { |
||||
|
private String error; |
||||
|
private Task task; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,31 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.queue.provider; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.job.JobType; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.TaskResultProto; |
||||
|
import org.thingsboard.server.queue.TbQueueConsumer; |
||||
|
import org.thingsboard.server.queue.TbQueueProducer; |
||||
|
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
||||
|
|
||||
|
public interface TaskProcessorQueueFactory { |
||||
|
|
||||
|
TbQueueConsumer<TbProtoQueueMsg<TaskProto>> createTaskConsumer(JobType jobType); |
||||
|
|
||||
|
TbQueueProducer<TbProtoQueueMsg<TaskResultProto>> createTaskResultProducer(); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,139 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.queue.task; |
||||
|
|
||||
|
import jakarta.annotation.PostConstruct; |
||||
|
import jakarta.annotation.PreDestroy; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
||||
|
import org.thingsboard.server.common.data.job.JobType; |
||||
|
import org.thingsboard.server.common.data.job.Task; |
||||
|
import org.thingsboard.server.common.data.job.TaskResult; |
||||
|
import org.thingsboard.server.common.data.job.TaskResult.TaskFailure; |
||||
|
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.TaskProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.TaskResultProto; |
||||
|
import org.thingsboard.server.queue.TbQueueCallback; |
||||
|
import org.thingsboard.server.queue.TbQueueConsumer; |
||||
|
import org.thingsboard.server.queue.TbQueueProducer; |
||||
|
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
||||
|
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; |
||||
|
import org.thingsboard.server.queue.provider.TaskProcessorQueueFactory; |
||||
|
import org.thingsboard.server.queue.util.AfterStartUp; |
||||
|
|
||||
|
import java.util.List; |
||||
|
import java.util.concurrent.ExecutorService; |
||||
|
import java.util.concurrent.Executors; |
||||
|
|
||||
|
@Slf4j |
||||
|
public abstract class TaskProcessor<T extends Task> { |
||||
|
|
||||
|
@Autowired |
||||
|
private TaskProcessorQueueFactory queueFactory; |
||||
|
|
||||
|
private QueueConsumerManager<TbProtoQueueMsg<TaskProto>> taskConsumer; |
||||
|
private TbQueueProducer<TbProtoQueueMsg<TaskResultProto>> taskResultProducer; |
||||
|
private ExecutorService consumerExecutor; |
||||
|
|
||||
|
@PostConstruct |
||||
|
public void init() { |
||||
|
consumerExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(getJobType().name().toLowerCase() + "-task-consumer")); |
||||
|
taskConsumer = QueueConsumerManager.<TbProtoQueueMsg<TaskProto>>builder() // fixme: should be consumer per partition
|
||||
|
.name(getJobType().name() + "-tasks") |
||||
|
.msgPackProcessor(this::processMsgs) |
||||
|
.pollInterval(125) |
||||
|
.consumerCreator(() -> queueFactory.createTaskConsumer(getJobType())) |
||||
|
.consumerExecutor(consumerExecutor) |
||||
|
.build(); |
||||
|
taskResultProducer = queueFactory.createTaskResultProducer(); |
||||
|
} |
||||
|
|
||||
|
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) |
||||
|
public void afterStartUp() { |
||||
|
taskConsumer.subscribe(); |
||||
|
taskConsumer.launch(); |
||||
|
} |
||||
|
|
||||
|
@PreDestroy |
||||
|
public void destroy() { |
||||
|
taskConsumer.stop(); |
||||
|
consumerExecutor.shutdownNow(); |
||||
|
} |
||||
|
|
||||
|
private void processMsgs(List<TbProtoQueueMsg<TaskProto>> msgs, TbQueueConsumer<TbProtoQueueMsg<TaskProto>> consumer) { |
||||
|
for (TbProtoQueueMsg<TaskProto> msg : msgs) { |
||||
|
TaskProto taskProto = msg.getValue(); |
||||
|
Task task = JacksonUtil.fromString(taskProto.getValue(), Task.class); |
||||
|
processTask((T) task); |
||||
|
} |
||||
|
consumer.commit(); |
||||
|
} |
||||
|
|
||||
|
private void processTask(T task) { |
||||
|
task.setAttempt(task.getAttempt() + 1); |
||||
|
log.info("Processing task: {}", task); |
||||
|
try { |
||||
|
process(task); |
||||
|
reportSuccess(task); |
||||
|
} catch (Exception e) { |
||||
|
log.error("Failed to process task (attempt {}): {}", task.getAttempt(), task, e); |
||||
|
if (task.getAttempt() < 3) { |
||||
|
processTask(task); |
||||
|
} else { |
||||
|
reportFailure(task, e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void reportSuccess(Task task) { |
||||
|
TaskResult result = TaskResult.builder() |
||||
|
.tenantId(task.getTenantId()) |
||||
|
.jobId(task.getJobId()) |
||||
|
.success(true) |
||||
|
.build(); |
||||
|
reportResult(result); |
||||
|
} |
||||
|
|
||||
|
private void reportFailure(Task task, Throwable error) { |
||||
|
TaskResult result = TaskResult.builder() |
||||
|
.tenantId(task.getTenantId()) |
||||
|
.jobId(task.getJobId()) |
||||
|
.failure(TaskFailure.builder() |
||||
|
.error(error.getMessage()) |
||||
|
.task(task) |
||||
|
.build()) |
||||
|
.build(); |
||||
|
reportResult(result); |
||||
|
} |
||||
|
|
||||
|
private void reportResult(TaskResult result) { |
||||
|
log.info("Reporting result: {}", result); |
||||
|
TaskResultProto resultProto = TaskResultProto.newBuilder() |
||||
|
.setValue(JacksonUtil.toString(result)) |
||||
|
.build(); |
||||
|
TbProtoQueueMsg<TaskResultProto> msg = new TbProtoQueueMsg<>(result.getJobId().getId(), resultProto); |
||||
|
taskResultProducer.send(TopicPartitionInfo.builder() |
||||
|
.topic(taskResultProducer.getDefaultTopic()) |
||||
|
.build(), msg, TbQueueCallback.EMPTY); |
||||
|
} |
||||
|
|
||||
|
protected abstract void process(T task) throws Exception; |
||||
|
|
||||
|
public abstract JobType getJobType(); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,96 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.dao.model.sql; |
||||
|
|
||||
|
import com.fasterxml.jackson.databind.JsonNode; |
||||
|
import jakarta.persistence.Column; |
||||
|
import jakarta.persistence.Convert; |
||||
|
import jakarta.persistence.Entity; |
||||
|
import jakarta.persistence.EnumType; |
||||
|
import jakarta.persistence.Enumerated; |
||||
|
import jakarta.persistence.Table; |
||||
|
import lombok.Data; |
||||
|
import lombok.EqualsAndHashCode; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import org.hibernate.annotations.JdbcType; |
||||
|
import org.hibernate.dialect.PostgreSQLJsonPGObjectJsonbType; |
||||
|
import org.thingsboard.server.common.data.id.JobId; |
||||
|
import org.thingsboard.server.common.data.job.Job; |
||||
|
import org.thingsboard.server.common.data.job.JobConfiguration; |
||||
|
import org.thingsboard.server.common.data.job.JobResult; |
||||
|
import org.thingsboard.server.common.data.job.JobStatus; |
||||
|
import org.thingsboard.server.common.data.job.JobType; |
||||
|
import org.thingsboard.server.dao.model.BaseSqlEntity; |
||||
|
import org.thingsboard.server.dao.model.ModelConstants; |
||||
|
import org.thingsboard.server.dao.util.mapping.JsonConverter; |
||||
|
|
||||
|
import java.util.UUID; |
||||
|
|
||||
|
@Data |
||||
|
@EqualsAndHashCode(callSuper = true) |
||||
|
@NoArgsConstructor |
||||
|
@Entity |
||||
|
@Table(name = ModelConstants.JOB_TABLE_NAME) |
||||
|
public class JobEntity extends BaseSqlEntity<Job> { |
||||
|
|
||||
|
@Column(name = ModelConstants.TENANT_ID_PROPERTY, nullable = false) |
||||
|
private UUID tenantId; |
||||
|
|
||||
|
@Enumerated(EnumType.STRING) |
||||
|
@Column(name = ModelConstants.JOB_TYPE_PROPERTY, nullable = false) |
||||
|
private JobType type; |
||||
|
|
||||
|
@Column(name = ModelConstants.JOB_KEY_PROPERTY, nullable = false) |
||||
|
private String key; |
||||
|
|
||||
|
@Enumerated(EnumType.STRING) |
||||
|
@Column(name = ModelConstants.JOB_STATUS_PROPERTY, nullable = false) |
||||
|
private JobStatus status; |
||||
|
|
||||
|
@Convert(converter = JsonConverter.class) |
||||
|
@Column(name = ModelConstants.JOB_CONFIGURATION_PROPERTY, nullable = false) |
||||
|
private JsonNode configuration; |
||||
|
|
||||
|
@Convert(converter = JsonConverter.class) |
||||
|
@JdbcType(PostgreSQLJsonPGObjectJsonbType.class) |
||||
|
@Column(name = ModelConstants.JOB_RESULT_PROPERTY) |
||||
|
private JsonNode result; |
||||
|
|
||||
|
public JobEntity(Job job) { |
||||
|
super(job); |
||||
|
this.tenantId = getTenantUuid(job.getTenantId()); |
||||
|
this.type = job.getType(); |
||||
|
this.key = job.getKey(); |
||||
|
this.status = job.getStatus(); |
||||
|
this.configuration = toJson(job.getConfiguration()); |
||||
|
this.result = toJson(job.getResult()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public Job toData() { |
||||
|
Job job = new Job(); |
||||
|
job.setId(new JobId(id)); |
||||
|
job.setCreatedTime(createdTime); |
||||
|
job.setTenantId(getTenantId(tenantId)); |
||||
|
job.setType(type); |
||||
|
job.setKey(key); |
||||
|
job.setStatus(status); |
||||
|
job.setConfiguration(fromJson(configuration, JobConfiguration.class)); |
||||
|
job.setResult(fromJson(result, JobResult.class)); |
||||
|
return job; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,69 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.dao.sql.task; |
||||
|
|
||||
|
import jakarta.transaction.Transactional; |
||||
|
import org.springframework.data.domain.Page; |
||||
|
import org.springframework.data.domain.Pageable; |
||||
|
import org.springframework.data.jpa.repository.JpaRepository; |
||||
|
import org.springframework.data.jpa.repository.Modifying; |
||||
|
import org.springframework.data.jpa.repository.Query; |
||||
|
import org.springframework.data.repository.query.Param; |
||||
|
import org.springframework.stereotype.Repository; |
||||
|
import org.thingsboard.server.dao.model.sql.JobEntity; |
||||
|
|
||||
|
import java.util.UUID; |
||||
|
|
||||
|
@Repository |
||||
|
public interface JobRepository extends JpaRepository<JobEntity, UUID> { |
||||
|
|
||||
|
Page<JobEntity> findByTenantId(UUID tenantId, Pageable pageable); |
||||
|
|
||||
|
@Modifying |
||||
|
@Transactional |
||||
|
@Query(value = """ |
||||
|
UPDATE job |
||||
|
SET result = jsonb_set( |
||||
|
result, |
||||
|
'{successfulCount}', |
||||
|
to_jsonb((result->>'successfulCount')::int + :count) |
||||
|
) |
||||
|
WHERE id = :jobId |
||||
|
RETURNING ((result->>'successfulCount')::int + :count) |
||||
|
+ (result->>'failedCount')::int = (result->>'totalCount')::int |
||||
|
""", nativeQuery = true) |
||||
|
boolean reportTaskSuccess(@Param("jobId") UUID jobId, @Param("count") int count); |
||||
|
|
||||
|
@Modifying |
||||
|
@Transactional |
||||
|
@Query(value = """ |
||||
|
UPDATE job |
||||
|
SET result = jsonb_set( |
||||
|
jsonb_set( |
||||
|
result, |
||||
|
'{failedCount}', |
||||
|
to_jsonb((result->>'failedCount')::int + 1) |
||||
|
), |
||||
|
ARRAY['failures', :taskKey], |
||||
|
to_jsonb(:error) |
||||
|
) |
||||
|
WHERE id = :jobId |
||||
|
RETURNING ((result->>'failedCount')::int + 1) + (result->>'successfulCount')::int |
||||
|
= (result->>'totalCount')::int |
||||
|
""", nativeQuery = true) |
||||
|
boolean reportTaskFailure(@Param("jobId") UUID jobId, @Param("taskKey") String taskKey, @Param("error") String error); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,72 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.dao.sql.task; |
||||
|
|
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import org.springframework.data.jpa.repository.JpaRepository; |
||||
|
import org.springframework.stereotype.Component; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.id.JobId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.job.Job; |
||||
|
import org.thingsboard.server.common.data.page.PageData; |
||||
|
import org.thingsboard.server.common.data.page.PageLink; |
||||
|
import org.thingsboard.server.dao.DaoUtil; |
||||
|
import org.thingsboard.server.dao.model.sql.JobEntity; |
||||
|
import org.thingsboard.server.dao.sql.JpaAbstractDao; |
||||
|
import org.thingsboard.server.dao.task.JobDao; |
||||
|
import org.thingsboard.server.dao.util.SqlDao; |
||||
|
|
||||
|
import java.util.UUID; |
||||
|
|
||||
|
@Component |
||||
|
@SqlDao |
||||
|
@RequiredArgsConstructor |
||||
|
public class JpaJobDao extends JpaAbstractDao<JobEntity, Job> implements JobDao { |
||||
|
|
||||
|
private final JobRepository jobRepository; |
||||
|
|
||||
|
@Override |
||||
|
public PageData<Job> findByTenantId(TenantId tenantId, PageLink pageLink) { |
||||
|
return DaoUtil.toPageData(jobRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean reportTaskSuccess(JobId jobId, int tasksCount) { |
||||
|
return jobRepository.reportTaskSuccess(jobId.getId(), tasksCount); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean reportTaskFailure(JobId jobId, String taskKey, String error) { |
||||
|
return jobRepository.reportTaskFailure(jobId.getId(), taskKey, error); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public EntityType getEntityType() { |
||||
|
return EntityType.JOB; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected Class<JobEntity> getEntityClass() { |
||||
|
return JobEntity.class; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected JpaRepository<JobEntity, UUID> getRepository() { |
||||
|
return jobRepository; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,86 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.dao.task; |
||||
|
|
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.server.common.data.id.JobId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.job.Job; |
||||
|
import org.thingsboard.server.common.data.job.JobResult; |
||||
|
import org.thingsboard.server.common.data.job.JobStatus; |
||||
|
import org.thingsboard.server.common.data.job.TaskResult; |
||||
|
import org.thingsboard.server.common.data.job.TaskResult.TaskFailure; |
||||
|
import org.thingsboard.server.common.data.page.PageData; |
||||
|
import org.thingsboard.server.common.data.page.PageLink; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
@Service |
||||
|
@RequiredArgsConstructor |
||||
|
@Slf4j |
||||
|
public class DefaultJobService implements JobService { |
||||
|
|
||||
|
private final JobDao jobDao; |
||||
|
|
||||
|
@Override |
||||
|
public Job createJob(TenantId tenantId, Job job) { |
||||
|
return jobDao.save(tenantId, job); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void reportTaskResults(JobId jobId, List<TaskResult> results) { |
||||
|
Job job = jobDao.findById(TenantId.SYS_TENANT_ID, jobId.getId()); |
||||
|
switch (job.getStatus()) { |
||||
|
case PENDING -> { |
||||
|
job.setStatus(JobStatus.RUNNING); |
||||
|
} |
||||
|
case CANCELLED, COMPLETED, FAILED -> { |
||||
|
// got some stale stats
|
||||
|
return; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
JobResult jobResult = job.getResult(); |
||||
|
for (TaskResult taskResult : results) { |
||||
|
if (taskResult.isSuccess()) { |
||||
|
jobResult.setSuccessfulCount(jobResult.getSuccessfulCount() + 1); |
||||
|
} else { |
||||
|
TaskFailure failure = taskResult.getFailure(); |
||||
|
String key = failure.getTask().getKey(); |
||||
|
jobResult.setFailedCount(jobResult.getFailedCount() + 1); |
||||
|
jobResult.getFailures().put(key, failure.getError()); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (jobResult.getSuccessfulCount() + jobResult.getFailedCount() >= jobResult.getTotalCount()) { |
||||
|
if (jobResult.getFailures().isEmpty()) { |
||||
|
job.setStatus(JobStatus.COMPLETED); |
||||
|
} else { |
||||
|
job.setStatus(JobStatus.FAILED); |
||||
|
} |
||||
|
} |
||||
|
log.info("Saving job {}", job); |
||||
|
jobDao.save(TenantId.SYS_TENANT_ID, job); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public PageData<Job> findJobsByTenantId(TenantId tenantId, PageLink pageLink) { |
||||
|
return jobDao.findByTenantId(tenantId, pageLink); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,33 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.dao.task; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.id.JobId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.job.Job; |
||||
|
import org.thingsboard.server.common.data.page.PageData; |
||||
|
import org.thingsboard.server.common.data.page.PageLink; |
||||
|
import org.thingsboard.server.dao.Dao; |
||||
|
|
||||
|
public interface JobDao extends Dao<Job> { |
||||
|
|
||||
|
PageData<Job> findByTenantId(TenantId tenantId, PageLink pageLink); |
||||
|
|
||||
|
boolean reportTaskSuccess(JobId jobId, int tasksCount); |
||||
|
|
||||
|
boolean reportTaskFailure(JobId jobId, String taskKey, String error); |
||||
|
|
||||
|
} |
||||
Loading…
Reference in new issue