Igor Kulikov
9 years ago
32 changed files with 830 additions and 420 deletions
@ -0,0 +1,93 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2017 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.install; |
||||
|
|
||||
|
import com.fasterxml.jackson.databind.ObjectMapper; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.apache.commons.csv.CSVFormat; |
||||
|
import org.apache.commons.csv.CSVParser; |
||||
|
import org.apache.commons.lang3.StringUtils; |
||||
|
import com.fasterxml.jackson.databind.JsonNode; |
||||
|
import org.thingsboard.server.common.data.UUIDConverter; |
||||
|
import org.thingsboard.server.common.data.id.CustomerId; |
||||
|
import org.thingsboard.server.common.data.id.DashboardId; |
||||
|
import org.thingsboard.server.dao.dashboard.DashboardService; |
||||
|
|
||||
|
import java.io.IOException; |
||||
|
import java.nio.file.Files; |
||||
|
import java.nio.file.Path; |
||||
|
import java.util.*; |
||||
|
|
||||
|
/** |
||||
|
* Created by igor on 2/27/18. |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public class DatabaseHelper { |
||||
|
|
||||
|
public static final CSVFormat CSV_DUMP_FORMAT = CSVFormat.DEFAULT.withNullString("\\N"); |
||||
|
|
||||
|
public static final String DEVICE = "device"; |
||||
|
public static final String TENANT_ID = "tenant_id"; |
||||
|
public static final String CUSTOMER_ID = "customer_id"; |
||||
|
public static final String SEARCH_TEXT = "search_text"; |
||||
|
public static final String ADDITIONAL_INFO = "additional_info"; |
||||
|
public static final String ASSET = "asset"; |
||||
|
public static final String DASHBOARD = "dashboard"; |
||||
|
public static final String ID = "id"; |
||||
|
public static final String TITLE = "title"; |
||||
|
public static final String ASSIGNED_CUSTOMERS = "assigned_customers"; |
||||
|
public static final String CONFIGURATION = "configuration"; |
||||
|
|
||||
|
public static final ObjectMapper objectMapper = new ObjectMapper(); |
||||
|
|
||||
|
public static void upgradeTo40_assignDashboards(Path dashboardsDump, DashboardService dashboardService, boolean sql) throws Exception { |
||||
|
String[] columns = new String[]{ID, TENANT_ID, CUSTOMER_ID, TITLE, SEARCH_TEXT, ASSIGNED_CUSTOMERS, CONFIGURATION}; |
||||
|
try (CSVParser csvParser = new CSVParser(Files.newBufferedReader(dashboardsDump), CSV_DUMP_FORMAT.withHeader(columns))) { |
||||
|
csvParser.forEach(record -> { |
||||
|
String customerIdString = record.get(CUSTOMER_ID); |
||||
|
String assignedCustomersString = record.get(ASSIGNED_CUSTOMERS); |
||||
|
DashboardId dashboardId = new DashboardId(toUUID(record.get(ID), sql)); |
||||
|
List<CustomerId> customerIds = new ArrayList<>(); |
||||
|
if (!StringUtils.isEmpty(assignedCustomersString)) { |
||||
|
try { |
||||
|
JsonNode assignedCustomersJson = objectMapper.readTree(assignedCustomersString); |
||||
|
Map<String,String> assignedCustomers = objectMapper.treeToValue(assignedCustomersJson, HashMap.class); |
||||
|
assignedCustomers.forEach((strCustomerId, title) -> { |
||||
|
customerIds.add(new CustomerId(UUID.fromString(strCustomerId))); |
||||
|
}); |
||||
|
} catch (IOException e) { |
||||
|
log.error("Unable to parse assigned customers field", e); |
||||
|
} |
||||
|
} |
||||
|
if (!StringUtils.isEmpty(customerIdString)) { |
||||
|
customerIds.add(new CustomerId(toUUID(customerIdString, sql))); |
||||
|
} |
||||
|
for (CustomerId customerId : customerIds) { |
||||
|
dashboardService.assignDashboardToCustomer(dashboardId, customerId); |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private static UUID toUUID(String src, boolean sql) { |
||||
|
if (sql) { |
||||
|
return UUIDConverter.fromString(src); |
||||
|
} else { |
||||
|
return UUID.fromString(src); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,146 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2017 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.install.sql; |
||||
|
|
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.apache.commons.csv.CSVParser; |
||||
|
import org.apache.commons.csv.CSVPrinter; |
||||
|
import org.apache.commons.csv.CSVRecord; |
||||
|
|
||||
|
import java.nio.file.Files; |
||||
|
import java.nio.file.Path; |
||||
|
import java.sql.*; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
import static org.thingsboard.server.service.install.DatabaseHelper.CSV_DUMP_FORMAT; |
||||
|
|
||||
|
/** |
||||
|
* Created by igor on 2/27/18. |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public class SqlDbHelper { |
||||
|
|
||||
|
public static Path dumpTableIfExists(Connection conn, String tableName, |
||||
|
String[] columns, String[] defaultValues, String dumpPrefix) throws Exception { |
||||
|
|
||||
|
DatabaseMetaData metaData = conn.getMetaData(); |
||||
|
ResultSet res = metaData.getTables(null, null, tableName, |
||||
|
new String[] {"TABLE"}); |
||||
|
if (res.next()) { |
||||
|
res.close(); |
||||
|
Path dumpFile = Files.createTempFile(dumpPrefix, null); |
||||
|
Files.deleteIfExists(dumpFile); |
||||
|
try (CSVPrinter csvPrinter = new CSVPrinter(Files.newBufferedWriter(dumpFile), CSV_DUMP_FORMAT)) { |
||||
|
try (PreparedStatement stmt = conn.prepareStatement("SELECT * FROM " + tableName)) { |
||||
|
try (ResultSet tableRes = stmt.executeQuery()) { |
||||
|
ResultSetMetaData resMetaData = tableRes.getMetaData(); |
||||
|
Map<String, Integer> columnIndexMap = new HashMap<>(); |
||||
|
for (int i = 0; i < resMetaData.getColumnCount(); i++) { |
||||
|
String columnName = resMetaData.getColumnName(i); |
||||
|
columnIndexMap.put(columnName, i); |
||||
|
} |
||||
|
while(tableRes.next()) { |
||||
|
dumpRow(tableRes, columnIndexMap, columns, defaultValues, csvPrinter); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return dumpFile; |
||||
|
} else { |
||||
|
return null; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public static void loadTable(Connection conn, String tableName, String[] columns, Path sourceFile) throws Exception { |
||||
|
PreparedStatement prepared = conn.prepareStatement(createInsertStatement(tableName, columns)); |
||||
|
prepared.getParameterMetaData(); |
||||
|
try (CSVParser csvParser = new CSVParser(Files.newBufferedReader(sourceFile), CSV_DUMP_FORMAT.withHeader(columns))) { |
||||
|
csvParser.forEach(record -> { |
||||
|
try { |
||||
|
for (int i=0;i<columns.length;i++) { |
||||
|
setColumnValue(i, columns[i], record, prepared); |
||||
|
} |
||||
|
prepared.execute(); |
||||
|
} catch (SQLException e) { |
||||
|
log.error("Unable to load table record!", e); |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private static void dumpRow(ResultSet res, Map<String, Integer> columnIndexMap, String[] columns, |
||||
|
String[] defaultValues, CSVPrinter csvPrinter) throws Exception { |
||||
|
List<String> record = new ArrayList<>(); |
||||
|
for (int i=0;i<columns.length;i++) { |
||||
|
String column = columns[i]; |
||||
|
String defaultValue; |
||||
|
if (defaultValues != null && i < defaultValues.length) { |
||||
|
defaultValue = defaultValues[i]; |
||||
|
} else { |
||||
|
defaultValue = ""; |
||||
|
} |
||||
|
record.add(getColumnValue(column, defaultValue, columnIndexMap, res)); |
||||
|
} |
||||
|
csvPrinter.printRecord(record); |
||||
|
} |
||||
|
|
||||
|
private static String getColumnValue(String column, String defaultValue, Map<String, Integer> columnIndexMap, ResultSet res) { |
||||
|
int index = columnIndexMap.containsKey(column) ? columnIndexMap.get(column) : -1; |
||||
|
if (index > -1) { |
||||
|
String str; |
||||
|
try { |
||||
|
Object obj = res.getObject(index); |
||||
|
if (obj == null) { |
||||
|
str = ""; |
||||
|
} else { |
||||
|
str = obj.toString(); |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
str = ""; |
||||
|
} |
||||
|
return str; |
||||
|
} else { |
||||
|
return defaultValue; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private static void setColumnValue(int index, String column, |
||||
|
CSVRecord record, PreparedStatement preparedStatement) throws SQLException { |
||||
|
String value = record.get(column); |
||||
|
int type = preparedStatement.getParameterMetaData().getParameterType(index + 1); |
||||
|
preparedStatement.setObject(index + 1, value, type); |
||||
|
} |
||||
|
|
||||
|
private static String createInsertStatement(String tableName, String[] columns) { |
||||
|
StringBuilder insertStatementBuilder = new StringBuilder(); |
||||
|
insertStatementBuilder.append("INSERT INTO ").append(tableName).append(" ("); |
||||
|
for (String column : columns) { |
||||
|
insertStatementBuilder.append(column).append(","); |
||||
|
} |
||||
|
insertStatementBuilder.deleteCharAt(insertStatementBuilder.length() - 1); |
||||
|
insertStatementBuilder.append(") VALUES ("); |
||||
|
for (String column : columns) { |
||||
|
insertStatementBuilder.append("?").append(","); |
||||
|
} |
||||
|
insertStatementBuilder.deleteCharAt(insertStatementBuilder.length() - 1); |
||||
|
insertStatementBuilder.append(")"); |
||||
|
return insertStatementBuilder.toString(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,50 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2017 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.dao.service; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.id.IdBased; |
||||
|
import org.thingsboard.server.common.data.page.TimePageLink; |
||||
|
|
||||
|
import java.sql.Time; |
||||
|
import java.util.List; |
||||
|
import java.util.UUID; |
||||
|
|
||||
|
public abstract class TimePaginatedRemover<I, D extends IdBased<?>> { |
||||
|
|
||||
|
private static final int DEFAULT_LIMIT = 100; |
||||
|
|
||||
|
public void removeEntities(I id) { |
||||
|
TimePageLink pageLink = new TimePageLink(DEFAULT_LIMIT); |
||||
|
boolean hasNext = true; |
||||
|
while (hasNext) { |
||||
|
List<D> entities = findEntities(id, pageLink); |
||||
|
for (D entity : entities) { |
||||
|
removeEntity(entity); |
||||
|
} |
||||
|
hasNext = entities.size() == pageLink.getLimit(); |
||||
|
if (hasNext) { |
||||
|
int index = entities.size() - 1; |
||||
|
UUID idOffset = entities.get(index).getUuidId(); |
||||
|
pageLink.setIdOffset(idOffset); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
protected abstract List<D> findEntities(I id, TimePageLink pageLink); |
||||
|
|
||||
|
protected abstract void removeEntity(D entity); |
||||
|
|
||||
|
} |
||||
Loading…
Reference in new issue