diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql new file mode 100644 index 0000000000..2559de38df --- /dev/null +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -0,0 +1,30 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE TABLE IF NOT EXISTS queue ( + id uuid NOT NULL CONSTRAINT queue_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid, + name varchar(255), + topic varchar(255), + poll_interval int, + partitions int, + consumer_per_partition boolean, + pack_processing_timeout bigint, + submit_strategy varchar(255), + processing_strategy varchar(255), + additional_info varchar +); diff --git a/application/src/main/data/upgrade/3.3.4/schema_update_device_profile.sql b/application/src/main/data/upgrade/3.3.4/schema_update_device_profile.sql new file mode 100644 index 0000000000..5f739e3a98 --- /dev/null +++ b/application/src/main/data/upgrade/3.3.4/schema_update_device_profile.sql @@ -0,0 +1,49 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +ALTER TABLE device_profile + ADD COLUMN IF NOT EXISTS default_queue_id uuid; + +DO +$$ + BEGIN + IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'fk_default_queue_device_profile') THEN + ALTER TABLE device_profile + ADD CONSTRAINT fk_default_queue_device_profile FOREIGN KEY (default_queue_id) REFERENCES queue (id); + END IF; + END; +$$; + +DO +$$ + BEGIN + IF EXISTS + (SELECT column_name + FROM information_schema.columns + WHERE table_name = 'device_profile' + AND column_name = 'default_queue_name' + ) + THEN + UPDATE device_profile + SET default_queue_id = q.id + FROM queue as q + WHERE default_queue_name = q.name; + END IF; + END +$$; + +ALTER TABLE device_profile + DROP COLUMN IF EXISTS default_queue_name; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 4c8e4333a6..5ed8f588e2 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -219,6 +219,7 @@ public class ThingsboardInstallService { databaseEntitiesUpgradeService.upgradeDatabase("3.3.3"); case "3.3.4": log.info("Upgrading ThingsBoard from version 3.3.4 to 3.4.0 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.3.4"); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index f7b9e5cad1..473f3accda 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -15,18 +15,31 @@ */ package org.thingsboard.server.service.install; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.queue.ProcessingStrategy; +import org.thingsboard.server.common.data.queue.ProcessingStrategyType; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.SubmitStrategy; +import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.service.install.sql.SqlDbHelper; @@ -43,7 +56,9 @@ import java.sql.SQLSyntaxErrorException; import java.sql.SQLWarning; import java.sql.Statement; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; import static org.thingsboard.server.service.install.DatabaseHelper.ASSIGNED_CUSTOMERS; @@ -101,6 +116,14 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService @Autowired private ApiUsageStateService apiUsageStateService; + @Autowired + private QueueService queueService; + + @Autowired + private TbRuleEngineQueueConfigService queueConfig; + + @Autowired + private RuleChainService ruleChainService; @Override public void upgradeDatabase(String fromVersion) throws Exception { @@ -534,6 +557,81 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.3.4": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL); + loadSql(schemaUpdateFile, conn); + + log.info("Loading queues..."); + try { + if (!CollectionUtils.isEmpty(queueConfig.getQueues())) { + queueConfig.getQueues().forEach(queueSettings -> { + Queue queue = new Queue(); + queue.setTenantId(TenantId.SYS_TENANT_ID); + queue.setName(queueSettings.getName()); + queue.setTopic(queueSettings.getTopic()); + queue.setPollInterval(queueSettings.getPollInterval()); + queue.setPartitions(queueSettings.getPartitions()); + queue.setPackProcessingTimeout(queueSettings.getPackProcessingTimeout()); + SubmitStrategy submitStrategy = new SubmitStrategy(); + submitStrategy.setBatchSize(queueSettings.getSubmitStrategy().getBatchSize()); + submitStrategy.setType(SubmitStrategyType.valueOf(queueSettings.getSubmitStrategy().getType())); + queue.setSubmitStrategy(submitStrategy); + ProcessingStrategy processingStrategy = new ProcessingStrategy(); + processingStrategy.setType(ProcessingStrategyType.valueOf(queueSettings.getProcessingStrategy().getType())); + processingStrategy.setRetries(queueSettings.getProcessingStrategy().getRetries()); + processingStrategy.setFailurePercentage(queueSettings.getProcessingStrategy().getFailurePercentage()); + processingStrategy.setPauseBetweenRetries(queueSettings.getProcessingStrategy().getPauseBetweenRetries()); + processingStrategy.setMaxPauseBetweenRetries(queueSettings.getProcessingStrategy().getMaxPauseBetweenRetries()); + queue.setProcessingStrategy(processingStrategy); + queue.setConsumerPerPartition(queueSettings.isConsumerPerPartition()); + queueService.saveQueue(queue); + }); + } else { + systemDataLoaderService.createQueues(); + } + } catch (Exception e) { + } + + log.info("Updating device profiles..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", "schema_update_device_profile.sql"); + loadSql(schemaUpdateFile, conn); + + log.info("Updating checkpoint rule nodes..."); + PageLink pageLink = new PageLink(100); + PageData pageData; + do { + pageData = tenantService.findTenants(pageLink); + for (Tenant tenant : pageData.getData()) { + TenantId tenantId = tenant.getId(); + Map queues = + queueService.findQueuesByTenantId(tenantId).stream().collect(Collectors.toMap(Queue::getName, Queue::getId)); + try { + List checkpointNodes = + ruleChainService.findRuleNodesByTenantIdAndType(tenantId, "org.thingsboard.rule.engine.flow.TbCheckpointNode"); + checkpointNodes.forEach(node -> { + ObjectNode configuration = (ObjectNode) node.getConfiguration(); + JsonNode queueNameNode = configuration.remove("queueName"); + if (queueNameNode != null) { + String queueName = queueNameNode.asText(); + configuration.put("queueId", queues.get(queueName).toString()); + ruleChainService.saveRuleNode(tenantId, node); + } + }); + } catch (Exception e) { + } + } + pageLink = pageLink.nextPageLink(); + } while (pageData.hasNext()); + + log.info("Updating schema settings..."); + conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004000;"); + log.info("Schema updated."); + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/TbRuleEngineQueueConfigService.java b/application/src/main/java/org/thingsboard/server/service/install/TbRuleEngineQueueConfigService.java new file mode 100644 index 0000000000..1125467082 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/TbRuleEngineQueueConfigService.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Slf4j +@Data +@EnableAutoConfiguration +@Configuration +@ConfigurationProperties(prefix = "queue.rule-engine") +@Profile("install") +public class TbRuleEngineQueueConfigService { + + private String topic; + private List queues; + + @PostConstruct + public void validate() { + queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> { + log.error("Main queue is not configured in thingsboard.yml"); + return new RuntimeException("No \"Main\" queue configured!"); + }); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java index c41eebdbb6..d099f4fb7c 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java @@ -68,6 +68,10 @@ public class DefaultCacheCleanupService implements CacheCleanupService { log.info("Clear cache to upgrade from version 3.3.3 to 3.3.4 ..."); clearAll(); break; + case "3.3.4": + log.info("Clear cache to upgrade from version 3.3.4 to 3.4.0 ..."); + clearCacheByName("deviceProfiles"); + break; default: //Do nothing, since cache cleanup is optional. } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueServiceDeprecated.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueServiceDeprecated.java deleted file mode 100644 index 2328893abe..0000000000 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueServiceDeprecated.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -import org.thingsboard.server.common.msg.queue.ServiceType; - -import java.util.Set; - -public interface TbQueueServiceDeprecated { - - Set getQueuesByServiceType(ServiceType serviceType); - - String resolve(ServiceType serviceType, String queueName); - -} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java index 5089a6286e..1d99534636 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java @@ -92,5 +92,7 @@ public interface RuleChainService { List findRuleNodesByTenantIdAndType(TenantId tenantId, String name, String toString); + List findRuleNodesByTenantIdAndType(TenantId tenantId, String type); + RuleNode saveRuleNode(TenantId tenantId, RuleNode ruleNode); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/DefaultTbQueueServiceDeprecated.java b/common/queue/src/main/java/org/thingsboard/server/queue/DefaultTbQueueServiceDeprecated.java deleted file mode 100644 index 8f4175dad1..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/DefaultTbQueueServiceDeprecated.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -import lombok.RequiredArgsConstructor; -import org.springframework.util.StringUtils; -import org.thingsboard.server.common.msg.queue.ServiceQueue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PostConstruct; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.Set; -import java.util.stream.Collectors; - -//@Service -@RequiredArgsConstructor -public class DefaultTbQueueServiceDeprecated implements TbQueueServiceDeprecated { - - private final TbQueueRuleEngineSettings ruleEngineSettings; - private Set ruleEngineQueues; - - @PostConstruct - public void init() { - ruleEngineQueues = ruleEngineSettings.getQueues().stream() - .map(TbRuleEngineQueueConfiguration::getName).collect(Collectors.toCollection(LinkedHashSet::new)); - } - - @Override - public Set getQueuesByServiceType(ServiceType type) { - if (type == ServiceType.TB_RULE_ENGINE) { - return ruleEngineQueues; - } else { - return Collections.emptySet(); - } - } - - @Override - public String resolve(ServiceType serviceType, String queueName) { - if (StringUtils.isEmpty(queueName) || !getQueuesByServiceType(serviceType).contains(queueName)) { - return ServiceQueue.MAIN; - } else { - return queueName; - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java index eed0b8a910..eee70e9e22 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -16,32 +16,14 @@ package org.thingsboard.server.queue.settings; import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import java.util.List; - -@Slf4j @Data -@EnableAutoConfiguration -@Configuration -@ConfigurationProperties(prefix = "queue.rule-engine") +@Component public class TbQueueRuleEngineSettings { + @Value("${queue.rule-engine.topic}") private String topic; - private List queues; - - @PostConstruct - public void validate() { - queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> { - log.error("Main queue is not configured in thingsboard.yml"); - return new RuntimeException("No \"Main\" queue configured!"); - }); - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 097779f33e..166d142baf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -668,6 +668,14 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC return ruleNodeDao.findRuleNodesByTenantIdAndType(tenantId, type, search); } + @Override + public List findRuleNodesByTenantIdAndType(TenantId tenantId, String type) { + log.trace("Executing findRuleNodes, tenantId [{}], type {}", tenantId, type); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + validateString(type, "Incorrect type of the rule node"); + return ruleNodeDao.findRuleNodesByTenantIdAndType(tenantId, type, ""); + } + @Override public RuleNode saveRuleNode(TenantId tenantId, RuleNode ruleNode) { return ruleNodeDao.save(tenantId, ruleNode); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java index 9bd51dfdf4..470a774b90 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java @@ -24,7 +24,6 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TbRelationTypes; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; -import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.TbMsg; @Slf4j @@ -44,12 +43,6 @@ public class TbCheckpointNode implements TbNode { @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { this.config = TbNodeUtils.convert(configuration, TbCheckpointNodeConfiguration.class); - if (config.getQueueId() == null) { - Queue foundQueue = ctx.getQueueService().findQueueByTenantIdAndName(ctx.getTenantId(), config.getQueueName()); - if (foundQueue != null) { - config.setQueueId(foundQueue.getId()); - } - } } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java index b2da9142d7..0546732874 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java @@ -22,14 +22,10 @@ import org.thingsboard.server.common.data.id.QueueId; @Data public class TbCheckpointNodeConfiguration implements NodeConfiguration { - private String queueName; - private QueueId queueId; @Override public TbCheckpointNodeConfiguration defaultConfiguration() { - TbCheckpointNodeConfiguration configuration = new TbCheckpointNodeConfiguration(); - configuration.setQueueName("HighPriority"); - return configuration; + return new TbCheckpointNodeConfiguration(); } }