Igor Kulikov
6 years ago
14 changed files with 555 additions and 465 deletions
@ -0,0 +1,74 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2021 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.client.tools.migrator; |
||||
|
|
||||
|
import org.apache.commons.io.FileUtils; |
||||
|
import org.apache.commons.io.LineIterator; |
||||
|
import org.apache.commons.lang3.StringUtils; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.io.IOException; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
public class DictionaryParser { |
||||
|
private Map<String, String> dictionaryParsed = new HashMap<>(); |
||||
|
|
||||
|
public DictionaryParser(File sourceFile) throws IOException { |
||||
|
parseDictionaryDump(FileUtils.lineIterator(sourceFile)); |
||||
|
} |
||||
|
|
||||
|
public String getKeyByKeyId(String keyId) { |
||||
|
return dictionaryParsed.get(keyId); |
||||
|
} |
||||
|
|
||||
|
private boolean isBlockFinished(String line) { |
||||
|
return StringUtils.isBlank(line) || line.equals("\\."); |
||||
|
} |
||||
|
|
||||
|
private boolean isBlockStarted(String line) { |
||||
|
return line.startsWith("COPY public.ts_kv_dictionary ("); |
||||
|
} |
||||
|
|
||||
|
private void parseDictionaryDump(LineIterator iterator) { |
||||
|
try { |
||||
|
String tempLine; |
||||
|
while (iterator.hasNext()) { |
||||
|
tempLine = iterator.nextLine(); |
||||
|
|
||||
|
if (isBlockStarted(tempLine)) { |
||||
|
processBlock(iterator); |
||||
|
} |
||||
|
} |
||||
|
} finally { |
||||
|
iterator.close(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void processBlock(LineIterator lineIterator) { |
||||
|
String tempLine; |
||||
|
String[] lineSplited; |
||||
|
while(lineIterator.hasNext()) { |
||||
|
tempLine = lineIterator.nextLine(); |
||||
|
if(isBlockFinished(tempLine)) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
lineSplited = tempLine.split("\t"); |
||||
|
dictionaryParsed.put(lineSplited[1], lineSplited[0]); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -1,179 +0,0 @@ |
|||||
/** |
|
||||
* Copyright © 2016-2021 The Thingsboard Authors |
|
||||
* |
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||
* you may not use this file except in compliance with the License. |
|
||||
* You may obtain a copy of the License at |
|
||||
* |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||
* |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
package org.thingsboard.client.tools.migrator; |
|
||||
|
|
||||
import com.google.common.collect.Lists; |
|
||||
import org.apache.cassandra.io.sstable.CQLSSTableWriter; |
|
||||
import org.apache.commons.io.FileUtils; |
|
||||
import org.apache.commons.io.LineIterator; |
|
||||
import org.apache.commons.lang3.StringUtils; |
|
||||
import org.apache.commons.lang3.math.NumberUtils; |
|
||||
|
|
||||
import java.io.File; |
|
||||
import java.io.IOException; |
|
||||
import java.util.ArrayList; |
|
||||
import java.util.Arrays; |
|
||||
import java.util.Date; |
|
||||
import java.util.List; |
|
||||
import java.util.UUID; |
|
||||
import java.util.stream.Collectors; |
|
||||
|
|
||||
public class PgCaLatestMigrator { |
|
||||
|
|
||||
private static final long LOG_BATCH = 1000000; |
|
||||
private static final long rowPerFile = 1000000; |
|
||||
|
|
||||
|
|
||||
private static long linesProcessed = 0; |
|
||||
private static long linesMigrated = 0; |
|
||||
private static long castErrors = 0; |
|
||||
private static long castedOk = 0; |
|
||||
|
|
||||
private static long currentWriterCount = 1; |
|
||||
private static CQLSSTableWriter currentTsWriter = null; |
|
||||
|
|
||||
public static void migrateLatest(File sourceFile, File outDir, boolean castStringsIfPossible) throws IOException { |
|
||||
long startTs = System.currentTimeMillis(); |
|
||||
long stepLineTs = System.currentTimeMillis(); |
|
||||
long stepOkLineTs = System.currentTimeMillis(); |
|
||||
LineIterator iterator = FileUtils.lineIterator(sourceFile); |
|
||||
currentTsWriter = WriterBuilder.getTsWriter(outDir); |
|
||||
|
|
||||
boolean isBlockStarted = false; |
|
||||
boolean isBlockFinished = false; |
|
||||
|
|
||||
String line; |
|
||||
while (iterator.hasNext()) { |
|
||||
if (linesProcessed++ % LOG_BATCH == 0) { |
|
||||
System.out.println(new Date() + " linesProcessed = " + linesProcessed + " in " + (System.currentTimeMillis() - stepLineTs) + " castOk " + castedOk + " castErr " + castErrors); |
|
||||
stepLineTs = System.currentTimeMillis(); |
|
||||
} |
|
||||
|
|
||||
line = iterator.nextLine(); |
|
||||
|
|
||||
if (isBlockFinished) { |
|
||||
break; |
|
||||
} |
|
||||
|
|
||||
if (!isBlockStarted) { |
|
||||
if (isBlockStarted(line)) { |
|
||||
System.out.println(); |
|
||||
System.out.println(); |
|
||||
System.out.println(line); |
|
||||
System.out.println(); |
|
||||
System.out.println(); |
|
||||
isBlockStarted = true; |
|
||||
} |
|
||||
continue; |
|
||||
} |
|
||||
|
|
||||
if (isBlockFinished(line)) { |
|
||||
isBlockFinished = true; |
|
||||
} else { |
|
||||
try { |
|
||||
List<String> raw = Arrays.stream(line.trim().split("\t")) |
|
||||
.map(String::trim) |
|
||||
.filter(StringUtils::isNotEmpty) |
|
||||
.collect(Collectors.toList()); |
|
||||
List<Object> values = toValues(raw); |
|
||||
|
|
||||
if (currentWriterCount == 0) { |
|
||||
System.out.println(new Date() + " close writer " + new Date()); |
|
||||
currentTsWriter.close(); |
|
||||
currentTsWriter = WriterBuilder.getLatestWriter(outDir); |
|
||||
} |
|
||||
|
|
||||
if (castStringsIfPossible) { |
|
||||
currentTsWriter.addRow(castToNumericIfPossible(values)); |
|
||||
} else { |
|
||||
currentTsWriter.addRow(values); |
|
||||
} |
|
||||
currentWriterCount++; |
|
||||
if (currentWriterCount >= rowPerFile) { |
|
||||
currentWriterCount = 0; |
|
||||
} |
|
||||
|
|
||||
if (linesMigrated++ % LOG_BATCH == 0) { |
|
||||
System.out.println(new Date() + " migrated = " + linesMigrated + " in " + (System.currentTimeMillis() - stepOkLineTs)); |
|
||||
stepOkLineTs = System.currentTimeMillis(); |
|
||||
} |
|
||||
} catch (Exception ex) { |
|
||||
System.out.println(ex.getMessage() + " -> " + line); |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
} |
|
||||
|
|
||||
long endTs = System.currentTimeMillis(); |
|
||||
System.out.println(); |
|
||||
System.out.println(new Date() + " Migrated rows " + linesMigrated + " in " + (endTs - startTs)); |
|
||||
|
|
||||
currentTsWriter.close(); |
|
||||
System.out.println(); |
|
||||
System.out.println("Finished migrate Latest Telemetry"); |
|
||||
} |
|
||||
|
|
||||
|
|
||||
private static List<Object> castToNumericIfPossible(List<Object> values) { |
|
||||
try { |
|
||||
if (values.get(6) != null && NumberUtils.isNumber(values.get(6).toString())) { |
|
||||
Double casted = NumberUtils.createDouble(values.get(6).toString()); |
|
||||
List<Object> numeric = Lists.newArrayList(); |
|
||||
numeric.addAll(values); |
|
||||
numeric.set(6, null); |
|
||||
numeric.set(8, casted); |
|
||||
castedOk++; |
|
||||
return numeric; |
|
||||
} |
|
||||
} catch (Throwable th) { |
|
||||
castErrors++; |
|
||||
} |
|
||||
return values; |
|
||||
} |
|
||||
|
|
||||
private static List<Object> toValues(List<String> raw) { |
|
||||
//expected Table structure:
|
|
||||
// COPY public.ts_kv_latest (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) FROM stdin;
|
|
||||
|
|
||||
|
|
||||
List<Object> result = new ArrayList<>(); |
|
||||
result.add(raw.get(0)); |
|
||||
result.add(fromString(raw.get(1))); |
|
||||
result.add(raw.get(2)); |
|
||||
|
|
||||
long ts = Long.parseLong(raw.get(3)); |
|
||||
result.add(ts); |
|
||||
|
|
||||
result.add(raw.get(4).equals("\\N") ? null : raw.get(4).equals("t") ? Boolean.TRUE : Boolean.FALSE); |
|
||||
result.add(raw.get(5).equals("\\N") ? null : raw.get(5)); |
|
||||
result.add(raw.get(6).equals("\\N") ? null : Long.parseLong(raw.get(6))); |
|
||||
result.add(raw.get(7).equals("\\N") ? null : Double.parseDouble(raw.get(7))); |
|
||||
return result; |
|
||||
} |
|
||||
|
|
||||
public static UUID fromString(String src) { |
|
||||
return UUID.fromString(src.substring(7, 15) + "-" + src.substring(3, 7) + "-1" |
|
||||
+ src.substring(0, 3) + "-" + src.substring(15, 19) + "-" + src.substring(19)); |
|
||||
} |
|
||||
|
|
||||
private static boolean isBlockStarted(String line) { |
|
||||
return line.startsWith("COPY public.ts_kv_latest"); |
|
||||
} |
|
||||
|
|
||||
private static boolean isBlockFinished(String line) { |
|
||||
return StringUtils.isBlank(line) || line.equals("\\."); |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,282 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2021 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.client.tools.migrator; |
||||
|
|
||||
|
import com.google.common.collect.Lists; |
||||
|
import org.apache.cassandra.io.sstable.CQLSSTableWriter; |
||||
|
import org.apache.commons.io.FileUtils; |
||||
|
import org.apache.commons.io.LineIterator; |
||||
|
import org.apache.commons.lang3.StringUtils; |
||||
|
import org.apache.commons.lang3.math.NumberUtils; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.io.IOException; |
||||
|
import java.time.Instant; |
||||
|
import java.time.LocalDateTime; |
||||
|
import java.time.ZoneOffset; |
||||
|
import java.time.temporal.ChronoUnit; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.Arrays; |
||||
|
import java.util.Date; |
||||
|
import java.util.HashSet; |
||||
|
import java.util.List; |
||||
|
import java.util.Set; |
||||
|
import java.util.UUID; |
||||
|
import java.util.function.Function; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
public class PgCaMigrator { |
||||
|
|
||||
|
private final long LOG_BATCH = 1000000; |
||||
|
private final long rowPerFile = 1000000; |
||||
|
|
||||
|
private long linesTsMigrated = 0; |
||||
|
private long linesLatestMigrated = 0; |
||||
|
private long castErrors = 0; |
||||
|
private long castedOk = 0; |
||||
|
|
||||
|
private long currentWriterCount = 1; |
||||
|
|
||||
|
private final File sourceFile; |
||||
|
private final boolean castStringIfPossible; |
||||
|
|
||||
|
private final RelatedEntitiesParser entityIdsAndTypes; |
||||
|
private final DictionaryParser keyParser; |
||||
|
private CQLSSTableWriter currentTsWriter; |
||||
|
private CQLSSTableWriter currentPartitionsWriter; |
||||
|
private CQLSSTableWriter currentTsLatestWriter; |
||||
|
private final Set<String> partitions = new HashSet<>(); |
||||
|
|
||||
|
private File outTsDir; |
||||
|
private File outTsLatestDir; |
||||
|
|
||||
|
public PgCaMigrator(File sourceFile, |
||||
|
File ourTsDir, |
||||
|
File outTsPartitionDir, |
||||
|
File outTsLatestDir, |
||||
|
RelatedEntitiesParser allEntityIdsAndTypes, |
||||
|
DictionaryParser dictionaryParser, |
||||
|
boolean castStringsIfPossible) { |
||||
|
this.sourceFile = sourceFile; |
||||
|
this.entityIdsAndTypes = allEntityIdsAndTypes; |
||||
|
this.keyParser = dictionaryParser; |
||||
|
this.castStringIfPossible = castStringsIfPossible; |
||||
|
if(outTsLatestDir != null) { |
||||
|
this.currentTsLatestWriter = WriterBuilder.getLatestWriter(outTsLatestDir); |
||||
|
this.outTsLatestDir = outTsLatestDir; |
||||
|
} |
||||
|
if(ourTsDir != null) { |
||||
|
this.currentTsWriter = WriterBuilder.getTsWriter(ourTsDir); |
||||
|
this.currentPartitionsWriter = WriterBuilder.getPartitionWriter(outTsPartitionDir); |
||||
|
this.outTsDir = ourTsDir; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void migrate() throws IOException { |
||||
|
boolean isTsDone = false; |
||||
|
boolean isLatestDone = false; |
||||
|
String line; |
||||
|
LineIterator iterator = FileUtils.lineIterator(this.sourceFile); |
||||
|
|
||||
|
try { |
||||
|
while(iterator.hasNext()) { |
||||
|
line = iterator.nextLine(); |
||||
|
if(!isLatestDone && isBlockLatestStarted(line)) { |
||||
|
System.out.println("START TO MIGRATE LATEST"); |
||||
|
long start = System.currentTimeMillis(); |
||||
|
processBlock(iterator, currentTsLatestWriter, outTsLatestDir, this::toValuesLatest); |
||||
|
System.out.println("TOTAL LINES MIGRATED: " + linesLatestMigrated + ", FORMING OF SSL FOR LATEST TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms."); |
||||
|
isLatestDone = true; |
||||
|
} |
||||
|
|
||||
|
if(!isTsDone && isBlockTsStarted(line)) { |
||||
|
System.out.println("START TO MIGRATE TS"); |
||||
|
long start = System.currentTimeMillis(); |
||||
|
processBlock(iterator, currentTsWriter, outTsDir, this::toValuesTs); |
||||
|
System.out.println("TOTAL LINES MIGRATED: " + linesTsMigrated + ", FORMING OF SSL FOR TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms."); |
||||
|
isTsDone = true; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
System.out.println("Partitions collected " + partitions.size()); |
||||
|
long startTs = System.currentTimeMillis(); |
||||
|
for (String partition : partitions) { |
||||
|
String[] split = partition.split("\\|"); |
||||
|
List<Object> values = Lists.newArrayList(); |
||||
|
values.add(split[0]); |
||||
|
values.add(UUID.fromString(split[1])); |
||||
|
values.add(split[2]); |
||||
|
values.add(Long.parseLong(split[3])); |
||||
|
currentPartitionsWriter.addRow(values); |
||||
|
} |
||||
|
|
||||
|
System.out.println(new Date() + " Migrated partitions " + partitions.size() + " in " + (System.currentTimeMillis() - startTs)); |
||||
|
|
||||
|
System.out.println(); |
||||
|
System.out.println("Finished migrate Telemetry"); |
||||
|
|
||||
|
} finally { |
||||
|
iterator.close(); |
||||
|
currentTsLatestWriter.close(); |
||||
|
currentTsWriter.close(); |
||||
|
currentPartitionsWriter.close(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void logLinesProcessed(long lines) { |
||||
|
if (lines % LOG_BATCH == 0) { |
||||
|
System.out.println(new Date() + " lines processed = " + lines + " in, castOk " + castedOk + " castErr " + castErrors); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void logLinesMigrated(long lines) { |
||||
|
if(lines % LOG_BATCH == 0) { |
||||
|
System.out.println(new Date() + " lines migrated = " + lines + " in, castOk " + castedOk + " castErr " + castErrors); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void addTypeIdKey(List<Object> result, List<String> raw) { |
||||
|
result.add(entityIdsAndTypes.getEntityType(raw.get(0))); |
||||
|
result.add(UUID.fromString(raw.get(0))); |
||||
|
result.add(keyParser.getKeyByKeyId(raw.get(1))); |
||||
|
} |
||||
|
|
||||
|
private void addPartitions(List<Object> result, List<String> raw) { |
||||
|
long ts = Long.parseLong(raw.get(2)); |
||||
|
long partition = toPartitionTs(ts); |
||||
|
result.add(partition); |
||||
|
result.add(ts); |
||||
|
} |
||||
|
|
||||
|
private void addTimeseries(List<Object> result, List<String> raw) { |
||||
|
result.add(Long.parseLong(raw.get(2))); |
||||
|
} |
||||
|
|
||||
|
private void addValues(List<Object> result, List<String> raw) { |
||||
|
result.add(raw.get(3).equals("\\N") ? null : raw.get(3).equals("t") ? Boolean.TRUE : Boolean.FALSE); |
||||
|
result.add(raw.get(4).equals("\\N") ? null : raw.get(4)); |
||||
|
result.add(raw.get(5).equals("\\N") ? null : Long.parseLong(raw.get(5))); |
||||
|
result.add(raw.get(6).equals("\\N") ? null : Double.parseDouble(raw.get(6))); |
||||
|
result.add(raw.get(7).equals("\\N") ? null : raw.get(7)); |
||||
|
} |
||||
|
|
||||
|
private List<Object> toValuesTs(List<String> raw) { |
||||
|
|
||||
|
logLinesMigrated(linesTsMigrated++); |
||||
|
|
||||
|
List<Object> result = new ArrayList<>(); |
||||
|
|
||||
|
addTypeIdKey(result, raw); |
||||
|
addPartitions(result, raw); |
||||
|
addValues(result, raw); |
||||
|
|
||||
|
processPartitions(result); |
||||
|
|
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
private List<Object> toValuesLatest(List<String> raw) { |
||||
|
logLinesMigrated(linesLatestMigrated++); |
||||
|
List<Object> result = new ArrayList<>(); |
||||
|
|
||||
|
addTypeIdKey(result, raw); |
||||
|
addTimeseries(result, raw); |
||||
|
addValues(result, raw); |
||||
|
|
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
private long toPartitionTs(long ts) { |
||||
|
LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); |
||||
|
return time.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1).toInstant(ZoneOffset.UTC).toEpochMilli(); |
||||
|
} |
||||
|
|
||||
|
private void processPartitions(List<Object> values) { |
||||
|
String key = values.get(0) + "|" + values.get(1) + "|" + values.get(2) + "|" + values.get(3); |
||||
|
partitions.add(key); |
||||
|
} |
||||
|
|
||||
|
private void processBlock(LineIterator iterator, CQLSSTableWriter writer, File outDir, Function<List<String>, List<Object>> function) { |
||||
|
String currentLine; |
||||
|
long linesProcessed = 0; |
||||
|
while(iterator.hasNext()) { |
||||
|
logLinesProcessed(linesProcessed++); |
||||
|
currentLine = iterator.nextLine(); |
||||
|
if(isBlockFinished(currentLine)) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
try { |
||||
|
List<String> raw = Arrays.stream(currentLine.trim().split("\t")) |
||||
|
.map(String::trim) |
||||
|
.collect(Collectors.toList()); |
||||
|
List<Object> values = function.apply(raw); |
||||
|
|
||||
|
if (this.currentWriterCount == 0) { |
||||
|
System.out.println(new Date() + " close writer " + new Date()); |
||||
|
writer.close(); |
||||
|
writer = WriterBuilder.getLatestWriter(outDir); |
||||
|
} |
||||
|
|
||||
|
if (this.castStringIfPossible) { |
||||
|
writer.addRow(castToNumericIfPossible(values)); |
||||
|
} else { |
||||
|
writer.addRow(values); |
||||
|
} |
||||
|
|
||||
|
currentWriterCount++; |
||||
|
if (currentWriterCount >= rowPerFile) { |
||||
|
currentWriterCount = 0; |
||||
|
} |
||||
|
} catch (Exception ex) { |
||||
|
System.out.println(ex.getMessage() + " -> " + currentLine); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private List<Object> castToNumericIfPossible(List<Object> values) { |
||||
|
try { |
||||
|
if (values.get(6) != null && NumberUtils.isNumber(values.get(6).toString())) { |
||||
|
Double casted = NumberUtils.createDouble(values.get(6).toString()); |
||||
|
List<Object> numeric = Lists.newArrayList(); |
||||
|
numeric.addAll(values); |
||||
|
numeric.set(6, null); |
||||
|
numeric.set(8, casted); |
||||
|
castedOk++; |
||||
|
return numeric; |
||||
|
} |
||||
|
} catch (Throwable th) { |
||||
|
castErrors++; |
||||
|
} |
||||
|
|
||||
|
processPartitions(values); |
||||
|
|
||||
|
return values; |
||||
|
} |
||||
|
|
||||
|
private boolean isBlockFinished(String line) { |
||||
|
return StringUtils.isBlank(line) || line.equals("\\."); |
||||
|
} |
||||
|
|
||||
|
private boolean isBlockTsStarted(String line) { |
||||
|
return line.startsWith("COPY public.ts_kv ("); |
||||
|
} |
||||
|
|
||||
|
private boolean isBlockLatestStarted(String line) { |
||||
|
return line.startsWith("COPY public.ts_kv_latest ("); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -1,222 +0,0 @@ |
|||||
/** |
|
||||
* Copyright © 2016-2021 The Thingsboard Authors |
|
||||
* |
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||
* you may not use this file except in compliance with the License. |
|
||||
* You may obtain a copy of the License at |
|
||||
* |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||
* |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
package org.thingsboard.client.tools.migrator; |
|
||||
|
|
||||
import com.google.common.collect.Lists; |
|
||||
import org.apache.cassandra.io.sstable.CQLSSTableWriter; |
|
||||
import org.apache.commons.io.FileUtils; |
|
||||
import org.apache.commons.io.LineIterator; |
|
||||
import org.apache.commons.lang3.StringUtils; |
|
||||
import org.apache.commons.lang3.math.NumberUtils; |
|
||||
|
|
||||
import java.io.File; |
|
||||
import java.io.IOException; |
|
||||
import java.time.Instant; |
|
||||
import java.time.LocalDateTime; |
|
||||
import java.time.ZoneOffset; |
|
||||
import java.time.temporal.ChronoUnit; |
|
||||
import java.util.ArrayList; |
|
||||
import java.util.Arrays; |
|
||||
import java.util.Date; |
|
||||
import java.util.HashSet; |
|
||||
import java.util.List; |
|
||||
import java.util.Set; |
|
||||
import java.util.UUID; |
|
||||
import java.util.stream.Collectors; |
|
||||
|
|
||||
public class PostgresToCassandraTelemetryMigrator { |
|
||||
|
|
||||
private static final long LOG_BATCH = 1000000; |
|
||||
private static final long rowPerFile = 1000000; |
|
||||
|
|
||||
|
|
||||
private static long linesProcessed = 0; |
|
||||
private static long linesMigrated = 0; |
|
||||
private static long castErrors = 0; |
|
||||
private static long castedOk = 0; |
|
||||
|
|
||||
private static long currentWriterCount = 1; |
|
||||
private static CQLSSTableWriter currentTsWriter = null; |
|
||||
private static CQLSSTableWriter currentPartitionWriter = null; |
|
||||
|
|
||||
private static Set<String> partitions = new HashSet<>(); |
|
||||
|
|
||||
|
|
||||
public static void migrateTs(File sourceFile, File outTsDir, File outPartitionDir, boolean castStringsIfPossible) throws IOException { |
|
||||
long startTs = System.currentTimeMillis(); |
|
||||
long stepLineTs = System.currentTimeMillis(); |
|
||||
long stepOkLineTs = System.currentTimeMillis(); |
|
||||
LineIterator iterator = FileUtils.lineIterator(sourceFile); |
|
||||
currentTsWriter = WriterBuilder.getTsWriter(outTsDir); |
|
||||
currentPartitionWriter = WriterBuilder.getPartitionWriter(outPartitionDir); |
|
||||
|
|
||||
boolean isBlockStarted = false; |
|
||||
boolean isBlockFinished = false; |
|
||||
|
|
||||
String line; |
|
||||
while (iterator.hasNext()) { |
|
||||
if (linesProcessed++ % LOG_BATCH == 0) { |
|
||||
System.out.println(new Date() + " linesProcessed = " + linesProcessed + " in " + (System.currentTimeMillis() - stepLineTs) + " castOk " + castedOk + " castErr " + castErrors); |
|
||||
stepLineTs = System.currentTimeMillis(); |
|
||||
} |
|
||||
|
|
||||
line = iterator.nextLine(); |
|
||||
|
|
||||
if (isBlockFinished) { |
|
||||
break; |
|
||||
} |
|
||||
|
|
||||
if (!isBlockStarted) { |
|
||||
if (isBlockStarted(line)) { |
|
||||
System.out.println(); |
|
||||
System.out.println(); |
|
||||
System.out.println(line); |
|
||||
System.out.println(); |
|
||||
System.out.println(); |
|
||||
isBlockStarted = true; |
|
||||
} |
|
||||
continue; |
|
||||
} |
|
||||
|
|
||||
if (isBlockFinished(line)) { |
|
||||
isBlockFinished = true; |
|
||||
} else { |
|
||||
try { |
|
||||
List<String> raw = Arrays.stream(line.trim().split("\t")) |
|
||||
.map(String::trim) |
|
||||
.filter(StringUtils::isNotEmpty) |
|
||||
.collect(Collectors.toList()); |
|
||||
List<Object> values = toValues(raw); |
|
||||
|
|
||||
if (currentWriterCount == 0) { |
|
||||
System.out.println(new Date() + " close writer " + new Date()); |
|
||||
currentTsWriter.close(); |
|
||||
currentTsWriter = WriterBuilder.getTsWriter(outTsDir); |
|
||||
} |
|
||||
|
|
||||
if (castStringsIfPossible) { |
|
||||
currentTsWriter.addRow(castToNumericIfPossible(values)); |
|
||||
} else { |
|
||||
currentTsWriter.addRow(values); |
|
||||
} |
|
||||
processPartitions(values); |
|
||||
currentWriterCount++; |
|
||||
if (currentWriterCount >= rowPerFile) { |
|
||||
currentWriterCount = 0; |
|
||||
} |
|
||||
|
|
||||
if (linesMigrated++ % LOG_BATCH == 0) { |
|
||||
System.out.println(new Date() + " migrated = " + linesMigrated + " in " + (System.currentTimeMillis() - stepOkLineTs) + " partitions = " + partitions.size()); |
|
||||
stepOkLineTs = System.currentTimeMillis(); |
|
||||
} |
|
||||
} catch (Exception ex) { |
|
||||
System.out.println(ex.getMessage() + " -> " + line); |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
} |
|
||||
|
|
||||
long endTs = System.currentTimeMillis(); |
|
||||
System.out.println(); |
|
||||
System.out.println(new Date() + " Migrated rows " + linesMigrated + " in " + (endTs - startTs)); |
|
||||
System.out.println("Partitions collected " + partitions.size()); |
|
||||
|
|
||||
startTs = System.currentTimeMillis(); |
|
||||
for (String partition : partitions) { |
|
||||
String[] split = partition.split("\\|"); |
|
||||
List<Object> values = Lists.newArrayList(); |
|
||||
values.add(split[0]); |
|
||||
values.add(UUID.fromString(split[1])); |
|
||||
values.add(split[2]); |
|
||||
values.add(Long.parseLong(split[3])); |
|
||||
currentPartitionWriter.addRow(values); |
|
||||
} |
|
||||
currentPartitionWriter.close(); |
|
||||
endTs = System.currentTimeMillis(); |
|
||||
System.out.println(); |
|
||||
System.out.println(); |
|
||||
System.out.println(new Date() + " Migrated partitions " + partitions.size() + " in " + (endTs - startTs)); |
|
||||
|
|
||||
|
|
||||
currentTsWriter.close(); |
|
||||
System.out.println(); |
|
||||
System.out.println("Finished migrate Telemetry"); |
|
||||
} |
|
||||
|
|
||||
private static List<Object> castToNumericIfPossible(List<Object> values) { |
|
||||
try { |
|
||||
if (values.get(6) != null && NumberUtils.isNumber(values.get(6).toString())) { |
|
||||
Double casted = NumberUtils.createDouble(values.get(6).toString()); |
|
||||
List<Object> numeric = Lists.newArrayList(); |
|
||||
numeric.addAll(values); |
|
||||
numeric.set(6, null); |
|
||||
numeric.set(8, casted); |
|
||||
castedOk++; |
|
||||
return numeric; |
|
||||
} |
|
||||
} catch (Throwable th) { |
|
||||
castErrors++; |
|
||||
} |
|
||||
return values; |
|
||||
} |
|
||||
|
|
||||
private static void processPartitions(List<Object> values) { |
|
||||
String key = values.get(0) + "|" + values.get(1) + "|" + values.get(2) + "|" + values.get(3); |
|
||||
partitions.add(key); |
|
||||
} |
|
||||
|
|
||||
private static List<Object> toValues(List<String> raw) { |
|
||||
//expected Table structure:
|
|
||||
// COPY public.ts_kv (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) FROM stdin;
|
|
||||
|
|
||||
|
|
||||
List<Object> result = new ArrayList<>(); |
|
||||
result.add(raw.get(0)); |
|
||||
result.add(fromString(raw.get(1))); |
|
||||
result.add(raw.get(2)); |
|
||||
|
|
||||
long ts = Long.parseLong(raw.get(3)); |
|
||||
long partition = toPartitionTs(ts); |
|
||||
result.add(partition); |
|
||||
result.add(ts); |
|
||||
|
|
||||
result.add(raw.get(4).equals("\\N") ? null : raw.get(4).equals("t") ? Boolean.TRUE : Boolean.FALSE); |
|
||||
result.add(raw.get(5).equals("\\N") ? null : raw.get(5)); |
|
||||
result.add(raw.get(6).equals("\\N") ? null : Long.parseLong(raw.get(6))); |
|
||||
result.add(raw.get(7).equals("\\N") ? null : Double.parseDouble(raw.get(7))); |
|
||||
return result; |
|
||||
} |
|
||||
|
|
||||
public static UUID fromString(String src) { |
|
||||
return UUID.fromString(src.substring(7, 15) + "-" + src.substring(3, 7) + "-1" |
|
||||
+ src.substring(0, 3) + "-" + src.substring(15, 19) + "-" + src.substring(19)); |
|
||||
} |
|
||||
|
|
||||
private static long toPartitionTs(long ts) { |
|
||||
LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); |
|
||||
return time.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1).toInstant(ZoneOffset.UTC).toEpochMilli(); |
|
||||
// return TsPartitionDate.MONTHS.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli();
|
|
||||
} |
|
||||
|
|
||||
private static boolean isBlockStarted(String line) { |
|
||||
return line.startsWith("COPY public.ts_kv"); |
|
||||
} |
|
||||
|
|
||||
private static boolean isBlockFinished(String line) { |
|
||||
return StringUtils.isBlank(line) || line.equals("\\."); |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
@ -0,0 +1,87 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2021 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.client.tools.migrator; |
||||
|
|
||||
|
import org.apache.commons.io.FileUtils; |
||||
|
import org.apache.commons.io.LineIterator; |
||||
|
import org.apache.commons.lang3.StringUtils; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.io.IOException; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
public class RelatedEntitiesParser { |
||||
|
private final Map<String, String> allEntityIdsAndTypes = new HashMap<>(); |
||||
|
|
||||
|
private final Map<String, EntityType> tableNameAndEntityType = Map.ofEntries( |
||||
|
Map.entry("COPY public.alarm ", EntityType.ALARM), |
||||
|
Map.entry("COPY public.asset ", EntityType.ASSET), |
||||
|
Map.entry("COPY public.customer ", EntityType.CUSTOMER), |
||||
|
Map.entry("COPY public.dashboard ", EntityType.DASHBOARD), |
||||
|
Map.entry("COPY public.device ", EntityType.DEVICE), |
||||
|
Map.entry("COPY public.rule_chain ", EntityType.RULE_CHAIN), |
||||
|
Map.entry("COPY public.rule_node ", EntityType.RULE_NODE), |
||||
|
Map.entry("COPY public.tenant ", EntityType.TENANT), |
||||
|
Map.entry("COPY public.tb_user ", EntityType.USER), |
||||
|
Map.entry("COPY public.entity_view ", EntityType.ENTITY_VIEW), |
||||
|
Map.entry("COPY public.widgets_bundle ", EntityType.WIDGETS_BUNDLE), |
||||
|
Map.entry("COPY public.widget_type ", EntityType.WIDGET_TYPE), |
||||
|
Map.entry("COPY public.tenant_profile ", EntityType.TENANT_PROFILE), |
||||
|
Map.entry("COPY public.device_profile ", EntityType.DEVICE_PROFILE), |
||||
|
Map.entry("COPY public.api_usage_state ", EntityType.API_USAGE_STATE) |
||||
|
); |
||||
|
|
||||
|
public RelatedEntitiesParser(File source) throws IOException { |
||||
|
processAllTables(FileUtils.lineIterator(source)); |
||||
|
} |
||||
|
|
||||
|
public String getEntityType(String uuid) { |
||||
|
return this.allEntityIdsAndTypes.get(uuid); |
||||
|
} |
||||
|
|
||||
|
private boolean isBlockFinished(String line) { |
||||
|
return StringUtils.isBlank(line) || line.equals("\\."); |
||||
|
} |
||||
|
|
||||
|
private void processAllTables(LineIterator lineIterator) { |
||||
|
String currentLine; |
||||
|
try { |
||||
|
while (lineIterator.hasNext()) { |
||||
|
currentLine = lineIterator.nextLine(); |
||||
|
for(Map.Entry<String, EntityType> entry : tableNameAndEntityType.entrySet()) { |
||||
|
if(currentLine.startsWith(entry.getKey())) { |
||||
|
processBlock(lineIterator, entry.getValue()); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} finally { |
||||
|
lineIterator.close(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void processBlock(LineIterator lineIterator, EntityType entityType) { |
||||
|
String currentLine; |
||||
|
while(lineIterator.hasNext()) { |
||||
|
currentLine = lineIterator.nextLine(); |
||||
|
if(isBlockFinished(currentLine)) { |
||||
|
return; |
||||
|
} |
||||
|
allEntityIdsAndTypes.put(currentLine.split("\t")[0], entityType.name()); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue