diff --git a/pom.xml b/pom.xml index beaedb9447..d1edef19fb 100755 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,8 @@ 4.10.0 4.0.5 4.3.1.0 - 3.11.9 + 3.11.10 + 3.11.0 1.2.7 28.2-jre 2.6.1 @@ -1095,6 +1096,11 @@ java-driver-query-builder ${cassandra.version} + + com.datastax.cassandra + cassandra-driver-core + ${cassandra-driver-core.version} + io.dropwizard.metrics metrics-jmx diff --git a/tools/pom.xml b/tools/pom.xml index e3df0e08d0..99f966b283 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -54,17 +54,14 @@ org.apache.cassandra cassandra-all - 3.11.10 com.datastax.cassandra cassandra-driver-core - 3.10.1 commons-io commons-io - 2.5 diff --git a/tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java b/tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java index afa341f880..7aec7ba5f3 100644 --- a/tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java +++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.client.tools.migrator; import com.google.common.collect.Lists; @@ -122,29 +137,52 @@ public class PgCaMigrator { } } - private void logLinesProcessed() { - if (linesProcessed++ % LOG_BATCH == 0) { - System.out.println(new Date() + " linesProcessed = " + linesProcessed + " in, castOk " + castedOk + " castErr " + castErrors); + private void logLinesProcessed(long lines) { + if (lines % LOG_BATCH == 0) { + System.out.println(new Date() + " lines processed = " + lines + " in, castOk " + castedOk + " castErr " + castErrors); } } - private List toValuesTs(List raw) { - linesTsMigrated++; - List result = new ArrayList<>(); + private void logLinesMigrated(long lines) { + if(lines % LOG_BATCH == 0) { + System.out.println(new Date() + " lines migrated = " + lines + " in, castOk " + castedOk + " castErr " + castErrors); + } + } + + private void addTypeIdKey(List result, List raw) { result.add(entityIdsAndTypes.getEntityType(raw.get(0))); result.add(UUID.fromString(raw.get(0))); result.add(keyParser.getKeyByKeyId(raw.get(1))); + } + private void addPartitions(List result, List raw) { long ts = Long.parseLong(raw.get(2)); long partition = toPartitionTs(ts); result.add(partition); result.add(ts); + } + + private void addTimeseries(List result, List raw) { + result.add(Long.parseLong(raw.get(2))); + } + private void addValues(List result, List raw) { result.add(raw.get(3).equals("\\N") ? null : raw.get(3).equals("t") ? Boolean.TRUE : Boolean.FALSE); result.add(raw.get(4).equals("\\N") ? null : raw.get(4)); result.add(raw.get(5).equals("\\N") ? null : Long.parseLong(raw.get(5))); result.add(raw.get(6).equals("\\N") ? null : Double.parseDouble(raw.get(6))); result.add(raw.get(7).equals("\\N") ? null : raw.get(7)); + } + + private List toValuesTs(List raw) { + + logLinesMigrated(linesTsMigrated++); + + List result = new ArrayList<>(); + + addTypeIdKey(result, raw); + addPartitions(result, raw); + addValues(result, raw); processPartitions(result); @@ -152,20 +190,12 @@ public class PgCaMigrator { } private List toValuesLatest(List raw) { - linesLatestMigrated++; + logLinesMigrated(linesLatestMigrated++); List result = new ArrayList<>(); - result.add(this.entityIdsAndTypes.getEntityType(raw.get(0))); - result.add(UUID.fromString(raw.get(0))); - result.add(this.keyParser.getKeyByKeyId(raw.get(1))); - - long ts = Long.parseLong(raw.get(2)); - result.add(3, ts); - result.add(raw.get(3).equals("\\N") ? null : raw.get(3).equals("t") ? Boolean.TRUE : Boolean.FALSE); - result.add(raw.get(4).equals("\\N") ? null : raw.get(4)); - result.add(raw.get(5).equals("\\N") ? null : Long.parseLong(raw.get(5))); - result.add(raw.get(6).equals("\\N") ? null : Double.parseDouble(raw.get(6))); - result.add(raw.get(7).equals("\\N") ? null : raw.get(7)); + addTypeIdKey(result, raw); + addTimeseries(result, raw); + addValues(result, raw); return result; } @@ -184,7 +214,7 @@ public class PgCaMigrator { String currentLine; linesProcessed = 0; while(iterator.hasNext()) { - logLinesProcessed(); + logLinesProcessed(linesProcessed++); currentLine = iterator.nextLine(); if(isBlockFinished(currentLine)) { return; diff --git a/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md b/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md index 1c5a6d54b7..cfe1675eb2 100644 --- a/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md +++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md @@ -21,8 +21,6 @@ It will generate single jar file with all required dependencies inside `target d #### Dump data from the source Postgres Database *Do not use compression if possible because Tool can only work with uncompressed file -*If you want to migrate just `ts_kv` without `ts_kv_latest` just don't dump an unnecessary table and when starting the tool don't use arguments (paths) for input dump and output files* - 1. Dump related tables that need to correct save telemetry `pg_dump -h localhost -U postgres -d thingsboard -t tenant -t customer -t user -t dashboard -t asset -t device -t alarm -t rule_chain -t rule_node -t entity_view -t widgets_bundle -t widget_type -t tenant_profile -t device_profile -t api_usage_state -t tb_user > related_entities.dmp` @@ -38,21 +36,26 @@ Tool use 3 different directories for saving SSTables - `ts_kv_cf`, `ts_kv_latest Create 3 empty directories. For example: - /home/ubunut/migration/ts - /home/ubunut/migration/ts_latest - /home/ubunut/migration/ts_partition + /home/user/migration/ts + /home/user/migration/ts_latest + /home/user/migration/ts_partition #### Run tool -*Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination + +**If you want to migrate just `ts_kv` without `ts_kv_latest` or vice versa don't use arguments (paths) for output files* + +**Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination* ``` java -jar ./tools-2.4.1-SNAPSHOT-jar-with-dependencies.jar - -telemetryFrom ./source/ts_kv_all.dmp + -telemetryFrom /home/user/dump/ts_kv_all.dmp + -relatedEntities /home/user/dump/relatedEntities.dmp -latestOut /home/ubunut/migration/ts_latest - -tsOut /home/ubunut/migration/ts - -partitionsOut /home/ubunut/migration/ts_partition - -castEnable false + -tsOut /home/ubunut/migration/ts + -partitionsOut /home/ubunut/migration/ts_partition + -castEnable false ``` +*Use your paths for program arguments* Tool execution time depends on DB size, CPU resources and Disk throughput @@ -62,20 +65,21 @@ Tool execution time depends on DB size, CPU resources and Disk throughput 1. [Optional] install Cassandra on the instance 2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this file `schema-ts.cql` 3. Stop Cassandra -4. Copy generated SSTable files into cassandra data dir: +4. Look at `/var/lib/cassandra/data/thingsboard` and check for names of data folders +5. Copy generated SSTable files into cassandra data dir using next command: ``` - sudo find /home/ubunut/migration/ts -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13/ \; - sudo find /home/ubunut/migration/ts_latest -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_latest_cf-161449d0ee5511e9a5fa7d6f489ffd13/ \; - sudo find /home/ubunut/migration/ts_partition -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_partitions_cf-12e8fa80ee5511e9a5fa7d6f489ffd13/ \; + sudo find /home/user/migration/ts -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13/ \; + sudo find /home/user/migration/ts_latest -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_latest_cf-161449d0ee5511e9a5fa7d6f489ffd13/ \; + sudo find /home/user/migration/ts_partition -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_partitions_cf-12e8fa80ee5511e9a5fa7d6f489ffd13/ \; ``` - -5. Start Cassandra service and trigger compaction + *Pay attention! Data folders have similar name `ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13`, but you have to use own* +6. Start Cassandra service and trigger compaction + + Trigger compactions: `nodetool compact thingsboard` + + Check compaction status: `nodetool compactionstats` -``` - trigger compactions: nodetool compact thingsboard - check compaction status: nodetool compactionstats -``` ## Switch Thignsboard into Hybrid Mode