tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Code cleaning, improvement of README.md

pull/4287/head
AndrewVolosytnykhThingsboard 5 years ago
committed by Andrew Shvayka
parent
1240099c81
commit
d5658d5b32
4 changed files with 80 additions and 43 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      pom.xml
  2. 3
      tools/pom.xml
  3. 68
      tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java
  4. 44
      tools/src/main/java/org/thingsboard/client/tools/migrator/README.md

8
pom.xml
View File

@ -54,7 +54,8 @@
<cassandra.version>4.10.0</cassandra.version>
<metrics.version>4.0.5</metrics.version>
<cassandra-unit.version>4.3.1.0</cassandra-unit.version>
<cassandra-all.version>3.11.9</cassandra-all.version>
<cassandra-all.version>3.11.10</cassandra-all.version>
<cassandra-driver-core.version>3.11.0</cassandra-driver-core.version>
<takari-cpsuite.version>1.2.7</takari-cpsuite.version>
<guava.version>28.2-jre</guava.version>
<caffeine.version>2.6.1</caffeine.version>
@ -1095,6 +1096,11 @@
<artifactId>java-driver-query-builder</artifactId>
<version>${cassandra.version}</version>
</dependency>
<dependency>
<groupId>com.datastax.cassandra</groupId>
<artifactId>cassandra-driver-core</artifactId>
<version>${cassandra-driver-core.version}</version>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-jmx</artifactId>

3
tools/pom.xml
View File

@ -54,17 +54,14 @@
<dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-all</artifactId>
<version>3.11.10</version>
</dependency>
<dependency>
<groupId>com.datastax.cassandra</groupId>
<artifactId>cassandra-driver-core</artifactId>
<version>3.10.1</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
</dependencies>

68
tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java
View File

@ -1,3 +1,18 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.client.tools.migrator;
import com.google.common.collect.Lists;
@ -122,29 +137,52 @@ public class PgCaMigrator {
}
}
private void logLinesProcessed() {
if (linesProcessed++ % LOG_BATCH == 0) {
System.out.println(new Date() + " linesProcessed = " + linesProcessed + " in, castOk " + castedOk + " castErr " + castErrors);
private void logLinesProcessed(long lines) {
if (lines % LOG_BATCH == 0) {
System.out.println(new Date() + " lines processed = " + lines + " in, castOk " + castedOk + " castErr " + castErrors);
}
}
private List<Object> toValuesTs(List<String> raw) {
linesTsMigrated++;
List<Object> result = new ArrayList<>();
private void logLinesMigrated(long lines) {
if(lines % LOG_BATCH == 0) {
System.out.println(new Date() + " lines migrated = " + lines + " in, castOk " + castedOk + " castErr " + castErrors);
}
}
private void addTypeIdKey(List<Object> result, List<String> raw) {
result.add(entityIdsAndTypes.getEntityType(raw.get(0)));
result.add(UUID.fromString(raw.get(0)));
result.add(keyParser.getKeyByKeyId(raw.get(1)));
}
private void addPartitions(List<Object> result, List<String> raw) {
long ts = Long.parseLong(raw.get(2));
long partition = toPartitionTs(ts);
result.add(partition);
result.add(ts);
}
private void addTimeseries(List<Object> result, List<String> raw) {
result.add(Long.parseLong(raw.get(2)));
}
private void addValues(List<Object> result, List<String> raw) {
result.add(raw.get(3).equals("\\N") ? null : raw.get(3).equals("t") ? Boolean.TRUE : Boolean.FALSE);
result.add(raw.get(4).equals("\\N") ? null : raw.get(4));
result.add(raw.get(5).equals("\\N") ? null : Long.parseLong(raw.get(5)));
result.add(raw.get(6).equals("\\N") ? null : Double.parseDouble(raw.get(6)));
result.add(raw.get(7).equals("\\N") ? null : raw.get(7));
}
private List<Object> toValuesTs(List<String> raw) {
logLinesMigrated(linesTsMigrated++);
List<Object> result = new ArrayList<>();
addTypeIdKey(result, raw);
addPartitions(result, raw);
addValues(result, raw);
processPartitions(result);
@ -152,20 +190,12 @@ public class PgCaMigrator {
}
private List<Object> toValuesLatest(List<String> raw) {
linesLatestMigrated++;
logLinesMigrated(linesLatestMigrated++);
List<Object> result = new ArrayList<>();
result.add(this.entityIdsAndTypes.getEntityType(raw.get(0)));
result.add(UUID.fromString(raw.get(0)));
result.add(this.keyParser.getKeyByKeyId(raw.get(1)));
long ts = Long.parseLong(raw.get(2));
result.add(3, ts);
result.add(raw.get(3).equals("\\N") ? null : raw.get(3).equals("t") ? Boolean.TRUE : Boolean.FALSE);
result.add(raw.get(4).equals("\\N") ? null : raw.get(4));
result.add(raw.get(5).equals("\\N") ? null : Long.parseLong(raw.get(5)));
result.add(raw.get(6).equals("\\N") ? null : Double.parseDouble(raw.get(6)));
result.add(raw.get(7).equals("\\N") ? null : raw.get(7));
addTypeIdKey(result, raw);
addTimeseries(result, raw);
addValues(result, raw);
return result;
}
@ -184,7 +214,7 @@ public class PgCaMigrator {
String currentLine;
linesProcessed = 0;
while(iterator.hasNext()) {
logLinesProcessed();
logLinesProcessed(linesProcessed++);
currentLine = iterator.nextLine();
if(isBlockFinished(currentLine)) {
return;

44
tools/src/main/java/org/thingsboard/client/tools/migrator/README.md
View File

@ -21,8 +21,6 @@ It will generate single jar file with all required dependencies inside `target d
#### Dump data from the source Postgres Database
*Do not use compression if possible because Tool can only work with uncompressed file
*If you want to migrate just `ts_kv` without `ts_kv_latest` just don't dump an unnecessary table and when starting the tool don't use arguments (paths) for input dump and output files*
1. Dump related tables that need to correct save telemetry
`pg_dump -h localhost -U postgres -d thingsboard -t tenant -t customer -t user -t dashboard -t asset -t device -t alarm -t rule_chain -t rule_node -t entity_view -t widgets_bundle -t widget_type -t tenant_profile -t device_profile -t api_usage_state -t tb_user > related_entities.dmp`
@ -38,21 +36,26 @@ Tool use 3 different directories for saving SSTables - `ts_kv_cf`, `ts_kv_latest
Create 3 empty directories. For example:
/home/ubunut/migration/ts
/home/ubunut/migration/ts_latest
/home/ubunut/migration/ts_partition
/home/user/migration/ts
/home/user/migration/ts_latest
/home/user/migration/ts_partition
#### Run tool
*Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination
**If you want to migrate just `ts_kv` without `ts_kv_latest` or vice versa don't use arguments (paths) for output files*
**Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination*
```
java -jar ./tools-2.4.1-SNAPSHOT-jar-with-dependencies.jar
-telemetryFrom ./source/ts_kv_all.dmp
-telemetryFrom /home/user/dump/ts_kv_all.dmp
-relatedEntities /home/user/dump/relatedEntities.dmp
-latestOut /home/ubunut/migration/ts_latest
-tsOut /home/ubunut/migration/ts
-partitionsOut /home/ubunut/migration/ts_partition
-castEnable false
-tsOut /home/ubunut/migration/ts
-partitionsOut /home/ubunut/migration/ts_partition
-castEnable false
```
*Use your paths for program arguments*
Tool execution time depends on DB size, CPU resources and Disk throughput
@ -62,20 +65,21 @@ Tool execution time depends on DB size, CPU resources and Disk throughput
1. [Optional] install Cassandra on the instance
2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this file `schema-ts.cql`
3. Stop Cassandra
4. Copy generated SSTable files into cassandra data dir:
4. Look at `/var/lib/cassandra/data/thingsboard` and check for names of data folders
5. Copy generated SSTable files into cassandra data dir using next command:
```
sudo find /home/ubunut/migration/ts -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13/ \;
sudo find /home/ubunut/migration/ts_latest -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_latest_cf-161449d0ee5511e9a5fa7d6f489ffd13/ \;
sudo find /home/ubunut/migration/ts_partition -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_partitions_cf-12e8fa80ee5511e9a5fa7d6f489ffd13/ \;
sudo find /home/user/migration/ts -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13/ \;
sudo find /home/user/migration/ts_latest -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_latest_cf-161449d0ee5511e9a5fa7d6f489ffd13/ \;
sudo find /home/user/migration/ts_partition -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_partitions_cf-12e8fa80ee5511e9a5fa7d6f489ffd13/ \;
```
5. Start Cassandra service and trigger compaction
*Pay attention! Data folders have similar name `ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13`, but you have to use own*
6. Start Cassandra service and trigger compaction
Trigger compactions: `nodetool compact thingsboard`
Check compaction status: `nodetool compactionstats`
```
trigger compactions: nodetool compact thingsboard
check compaction status: nodetool compactionstats
```
## Switch Thignsboard into Hybrid Mode

Write Preview
Loading…
Cancel
Save