Browse Source
* optimized postgres sync * Fix consumer stopping when stopWhenRead enabled * Fix NPE on EDQS repartitioning * fixed EntityServiceTest * Fix EDQS yml props --------- Co-authored-by: ViacheslavKlimov <viacheslavklimov11@gmail.com>pull/12580/head
ViacheslavKlimov
2 years ago
27 changed files with 426 additions and 653 deletions
@ -1,539 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edqs; |
|||
|
|||
import com.fasterxml.jackson.databind.MappingIterator; |
|||
import com.fasterxml.jackson.dataformat.csv.CsvMapper; |
|||
import com.fasterxml.jackson.dataformat.csv.CsvSchema; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
|||
import org.thingsboard.server.common.data.ApiUsageState; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.Customer; |
|||
import org.thingsboard.server.common.data.Dashboard; |
|||
import org.thingsboard.server.common.data.Device; |
|||
import org.thingsboard.server.common.data.DeviceProfile; |
|||
import org.thingsboard.server.common.data.DeviceProfileType; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.EntityView; |
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
import org.thingsboard.server.common.data.StringUtils; |
|||
import org.thingsboard.server.common.data.Tenant; |
|||
import org.thingsboard.server.common.data.User; |
|||
import org.thingsboard.server.common.data.asset.Asset; |
|||
import org.thingsboard.server.common.data.asset.AssetProfile; |
|||
import org.thingsboard.server.common.data.converter.Converter; |
|||
import org.thingsboard.server.common.data.converter.ConverterType; |
|||
import org.thingsboard.server.common.data.edge.Edge; |
|||
import org.thingsboard.server.common.data.edqs.AttributeKv; |
|||
import org.thingsboard.server.common.data.edqs.LatestTsKv; |
|||
import org.thingsboard.server.common.data.group.EntityGroup; |
|||
import org.thingsboard.server.common.data.id.ApiUsageStateId; |
|||
import org.thingsboard.server.common.data.id.AssetId; |
|||
import org.thingsboard.server.common.data.id.AssetProfileId; |
|||
import org.thingsboard.server.common.data.id.ConverterId; |
|||
import org.thingsboard.server.common.data.id.CustomerId; |
|||
import org.thingsboard.server.common.data.id.DashboardId; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.DeviceProfileId; |
|||
import org.thingsboard.server.common.data.id.EdgeId; |
|||
import org.thingsboard.server.common.data.id.EntityGroupId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.EntityIdFactory; |
|||
import org.thingsboard.server.common.data.id.EntityViewId; |
|||
import org.thingsboard.server.common.data.id.IntegrationId; |
|||
import org.thingsboard.server.common.data.id.RoleId; |
|||
import org.thingsboard.server.common.data.id.RuleChainId; |
|||
import org.thingsboard.server.common.data.id.SchedulerEventId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.id.TenantProfileId; |
|||
import org.thingsboard.server.common.data.id.UserId; |
|||
import org.thingsboard.server.common.data.id.WidgetTypeId; |
|||
import org.thingsboard.server.common.data.id.WidgetsBundleId; |
|||
import org.thingsboard.server.common.data.integration.Integration; |
|||
import org.thingsboard.server.common.data.integration.IntegrationType; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BasicTsKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BooleanDataEntry; |
|||
import org.thingsboard.server.common.data.kv.DoubleDataEntry; |
|||
import org.thingsboard.server.common.data.kv.JsonDataEntry; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.data.kv.LongDataEntry; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.common.data.relation.EntityRelation; |
|||
import org.thingsboard.server.common.data.relation.RelationTypeGroup; |
|||
import org.thingsboard.server.common.data.role.Role; |
|||
import org.thingsboard.server.common.data.role.RoleType; |
|||
import org.thingsboard.server.common.data.rule.RuleChain; |
|||
import org.thingsboard.server.common.data.scheduler.SchedulerEvent; |
|||
import org.thingsboard.server.common.data.widget.WidgetType; |
|||
import org.thingsboard.server.common.data.widget.WidgetsBundle; |
|||
import org.thingsboard.server.common.msg.edqs.EdqsService; |
|||
import org.thingsboard.server.edqs.processor.EdqsConverter; |
|||
|
|||
import java.io.FileReader; |
|||
import java.util.Map; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.function.Consumer; |
|||
|
|||
import static org.thingsboard.common.util.JacksonUtil.toJsonNode; |
|||
|
|||
|
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
//@Service
|
|||
public class EdqsDataLoader { |
|||
|
|||
private final EdqsService edqsService; |
|||
private final EdqsConverter edqsConverter; |
|||
|
|||
public final static TenantId MAIN = TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")); |
|||
|
|||
private final String folder = "/home/viacheslav/Downloads/schwarz"; |
|||
|
|||
private ExecutorService executor = Executors.newFixedThreadPool(5, ThingsBoardThreadFactory.forName("edqs-publisher")); |
|||
|
|||
// @AfterStartUp(order = 100)
|
|||
public void load() throws Exception { |
|||
loadCustomers(); |
|||
loadDeviceProfile(); |
|||
loadDevices(); |
|||
loadAssets(); |
|||
loadEdges(); |
|||
loadEntityViews(); |
|||
loadTenants(); |
|||
loadUsers(); |
|||
loadDashboards(); |
|||
loadRuleChains(); |
|||
loadWidgetType(); |
|||
loadWidgetBundle(); |
|||
loadConverters(); |
|||
loadIntegrations(); |
|||
loadSchedulerEvents(); |
|||
loadRoles(); |
|||
loadApiUsageStates(); |
|||
loadAssetProfile(); |
|||
loadEntityGroups(); |
|||
loadRelations(); |
|||
|
|||
loadAttributes(); |
|||
loadTs(); |
|||
} |
|||
|
|||
private void loadCustomers() throws Exception { |
|||
load("customer.csv", (values) -> { |
|||
Customer customer = new Customer(); |
|||
customer.setTitle(values.get("title")); |
|||
customer.setId(new CustomerId(UUID.fromString(values.get("id")))); |
|||
customer.setCreatedTime(Long.parseLong(values.get("created_time"))); |
|||
customer.setTenantId(tenantId(values.get("tenant_id"))); |
|||
var parentCustomerId = values.get("parent_customer_id"); |
|||
if (StringUtils.isNotEmpty(parentCustomerId)) { |
|||
customer.setParentCustomerId(new CustomerId(UUID.fromString(parentCustomerId))); |
|||
} |
|||
edqsService.onUpdate(customer.getTenantId(), customer.getId(), customer); |
|||
}); |
|||
} |
|||
|
|||
private void loadDevices() throws Exception { |
|||
load("device.csv", (values) -> { |
|||
Device device = new Device(); |
|||
device.setType(values.get("type")); |
|||
device.setName(values.get("name")); |
|||
device.setLabel(values.get("label")); |
|||
device.setId(new DeviceId(uuid(values.get("id")))); |
|||
device.setCreatedTime(parseLong(values.get("created_time"))); |
|||
device.setCustomerId(customerId(values.get("customer_id"))); |
|||
device.setTenantId(tenantId(values.get("tenant_id"))); |
|||
device.setDeviceProfileId(new DeviceProfileId(uuid(values.get("device_profile_id")))); |
|||
device.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
|
|||
edqsService.onUpdate(device.getTenantId(), device.getId(), device); |
|||
}); |
|||
} |
|||
|
|||
private void loadAssets() throws Exception { |
|||
load("asset.csv", (values) -> { |
|||
Asset asset = new Asset(); |
|||
asset.setType(values.get("type")); |
|||
asset.setName(values.get("name")); |
|||
asset.setLabel(values.get("label")); |
|||
asset.setId(new AssetId(uuid(values.get("id")))); |
|||
asset.setCreatedTime(parseLong(values.get("created_time"))); |
|||
asset.setCustomerId(customerId(values.get("customer_id"))); |
|||
asset.setTenantId(tenantId(values.get("tenant_id"))); |
|||
asset.setAssetProfileId(new AssetProfileId(uuid(values.get("asset_profile_id")))); |
|||
asset.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
|
|||
edqsService.onUpdate(asset.getTenantId(), asset.getId(), asset); |
|||
}); |
|||
} |
|||
|
|||
private void loadEdges() throws Exception { |
|||
load("edge.csv", (values) -> { |
|||
Edge edge = new Edge(); |
|||
edge.setId(new EdgeId(uuid(values.get("id")))); |
|||
edge.setCreatedTime(parseLong(values.get("created_time"))); |
|||
edge.setType(values.get("type")); |
|||
edge.setName(values.get("name")); |
|||
edge.setLabel(values.get("label")); |
|||
edge.setCustomerId(customerId(values.get("customer_id"))); |
|||
edge.setTenantId(tenantId(values.get("tenant_id"))); |
|||
edge.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
|
|||
edqsService.onUpdate(edge.getTenantId(), edge.getId(), edge); |
|||
}); |
|||
} |
|||
|
|||
private void loadEntityViews() throws Exception { |
|||
load("entity_view.csv", (values) -> { |
|||
EntityView entityView = new EntityView(); |
|||
entityView.setId(new EntityViewId(uuid(values.get("id")))); |
|||
entityView.setCreatedTime(parseLong(values.get("created_time"))); |
|||
entityView.setType(values.get("type")); |
|||
entityView.setName(values.get("name")); |
|||
entityView.setCustomerId(customerId(values.get("customer_id"))); |
|||
entityView.setTenantId(tenantId(values.get("tenant_id"))); |
|||
entityView.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
|
|||
edqsService.onUpdate(entityView.getTenantId(), entityView.getId(), entityView); |
|||
}); |
|||
} |
|||
|
|||
private void loadTenants() throws Exception { |
|||
load("tenant.csv", (values) -> { |
|||
Tenant tenant = new Tenant(); |
|||
tenant.setId(new TenantId(uuid(values.get("id")))); |
|||
tenant.setCreatedTime(parseLong(values.get("created_time"))); |
|||
tenant.setEmail(values.get("email")); |
|||
tenant.setTitle(values.get("title")); |
|||
tenant.setCountry(values.get("country")); |
|||
tenant.setState(values.get("state")); |
|||
tenant.setCity(values.get("city")); |
|||
tenant.setAddress(values.get("address")); |
|||
tenant.setAddress2(values.get("address2")); |
|||
tenant.setZip(values.get("zip")); |
|||
tenant.setPhone(values.get("phone")); |
|||
tenant.setRegion(values.get("region")); |
|||
tenant.setTenantProfileId(new TenantProfileId(uuid(values.get("tenant_profile_id")))); |
|||
tenant.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
edqsService.onUpdate(MAIN, tenant.getId(), tenant); |
|||
}); |
|||
} |
|||
|
|||
private void loadUsers() throws Exception { |
|||
load("user.csv", (values) -> { |
|||
User user = new User(); |
|||
user.setId(new UserId(uuid(values.get("id")))); |
|||
user.setCreatedTime(parseLong(values.get("created_time"))); |
|||
user.setTenantId(tenantId(values.get("tenant_id"))); |
|||
user.setFirstName(values.get("first_name")); |
|||
user.setLastName(values.get("last_name")); |
|||
user.setEmail(values.get("email")); |
|||
user.setPhone(values.get("phone")); |
|||
user.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
|
|||
edqsService.onUpdate(user.getTenantId(), user.getId(), user); |
|||
}); |
|||
} |
|||
|
|||
private void loadDashboards() throws Exception { |
|||
load("dashboard.csv", (values) -> { |
|||
Dashboard dashboard = new Dashboard(); |
|||
dashboard.setId(new DashboardId(uuid(values.get("id")))); |
|||
dashboard.setCreatedTime(parseLong(values.get("created_time"))); |
|||
dashboard.setTenantId(tenantId(values.get("tenant_id"))); |
|||
dashboard.setTitle(values.get("title")); |
|||
|
|||
edqsService.onUpdate(dashboard.getTenantId(), dashboard.getId(), dashboard); |
|||
}); |
|||
} |
|||
|
|||
private void loadEntityGroups() throws Exception { |
|||
load("entity_group.csv", (values) -> { |
|||
EntityGroup entityGroup = new EntityGroup(); |
|||
entityGroup.setId(new EntityGroupId(uuid(values.get("id")))); |
|||
entityGroup.setCreatedTime(parseLong(values.get("created_time"))); |
|||
entityGroup.setName(values.get("name")); |
|||
entityGroup.setOwnerId(entityId(values.get("owner_type"), values.get("owner_id"))); |
|||
entityGroup.setType(EntityType.valueOf(values.get("type"))); |
|||
edqsService.onUpdate(MAIN, entityGroup.getId(), entityGroup); |
|||
}); |
|||
} |
|||
|
|||
private void loadRelations() throws Exception { |
|||
load("relation.csv", (values) -> { |
|||
EntityRelation entityRelation = new EntityRelation(); |
|||
entityRelation.setFrom(entityId(values.get("from_type"), values.get("from_id"))); |
|||
entityRelation.setTo(entityId(values.get("to_type"), values.get("to_id"))); |
|||
entityRelation.setTypeGroup(RelationTypeGroup.valueOf(values.get("relation_type_group"))); |
|||
entityRelation.setType(values.get("relation_type")); |
|||
edqsService.onUpdate(MAIN, ObjectType.RELATION, entityRelation); |
|||
}); |
|||
} |
|||
|
|||
private void loadRuleChains() throws Exception { |
|||
load("rule_chain.csv", (values) -> { |
|||
RuleChain ruleChain = new RuleChain(); |
|||
ruleChain.setId(new RuleChainId(uuid(values.get("id")))); |
|||
ruleChain.setCreatedTime(parseLong(values.get("created_time"))); |
|||
ruleChain.setName(values.get("name")); |
|||
ruleChain.setTenantId(tenantId(values.get("tenant_id"))); |
|||
ruleChain.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
|
|||
edqsService.onUpdate(ruleChain.getTenantId(), ruleChain.getId(), ruleChain); |
|||
}); |
|||
} |
|||
|
|||
private void loadWidgetType() throws Exception { |
|||
load("widget_type.csv", (values) -> { |
|||
WidgetType widgetType = new WidgetType(); |
|||
widgetType.setId(new WidgetTypeId(uuid(values.get("id")))); |
|||
widgetType.setCreatedTime(parseLong(values.get("created_time"))); |
|||
widgetType.setName(values.get("name")); |
|||
widgetType.setTenantId(tenantId(values.get("tenant_id"))); |
|||
|
|||
edqsService.onUpdate(widgetType.getTenantId(), widgetType.getId(), widgetType); |
|||
}); |
|||
} |
|||
|
|||
private void loadWidgetBundle() throws Exception { |
|||
load("widgets_bundle.csv", (values) -> { |
|||
WidgetsBundle widgetsBundle = new WidgetsBundle(); |
|||
widgetsBundle.setId(new WidgetsBundleId(uuid(values.get("id")))); |
|||
widgetsBundle.setCreatedTime(parseLong(values.get("created_time"))); |
|||
widgetsBundle.setTitle(values.get("title")); |
|||
widgetsBundle.setTenantId(tenantId(values.get("tenant_id"))); |
|||
|
|||
edqsService.onUpdate(widgetsBundle.getTenantId(), widgetsBundle.getId(), widgetsBundle); |
|||
}); |
|||
} |
|||
|
|||
private void loadConverters() throws Exception { |
|||
load("converter.csv", (values) -> { |
|||
Converter converter = new Converter(); |
|||
converter.setId(new ConverterId(uuid(values.get("id")))); |
|||
converter.setCreatedTime(parseLong(values.get("created_time"))); |
|||
converter.setName(values.get("name")); |
|||
converter.setType(ConverterType.valueOf(values.get("type"))); |
|||
converter.setTenantId(tenantId(values.get("tenant_id"))); |
|||
converter.setEdgeTemplate(parseBoolean(values.get("is_edge_template"))); |
|||
converter.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
|
|||
edqsService.onUpdate(converter.getTenantId(), converter.getId(), converter); |
|||
}); |
|||
} |
|||
|
|||
private void loadIntegrations() throws Exception { |
|||
load("integration.csv", (values) -> { |
|||
Integration integration = new Integration(); |
|||
integration.setId(new IntegrationId(uuid(values.get("id")))); |
|||
integration.setCreatedTime(parseLong(values.get("created_time"))); |
|||
integration.setName(values.get("name")); |
|||
integration.setType(IntegrationType.valueOf(values.get("type"))); |
|||
integration.setTenantId(tenantId(values.get("tenant_id"))); |
|||
integration.setEdgeTemplate(parseBoolean(values.get("is_edge_template"))); |
|||
integration.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
|
|||
edqsService.onUpdate(integration.getTenantId(), integration.getId(), integration); |
|||
}); |
|||
} |
|||
|
|||
private void loadSchedulerEvents() throws Exception { |
|||
load("scheduler_event.csv", (values) -> { |
|||
SchedulerEvent schedulerEvent = new SchedulerEvent(); |
|||
schedulerEvent.setId(new SchedulerEventId(uuid(values.get("id")))); |
|||
schedulerEvent.setCreatedTime(parseLong(values.get("created_time"))); |
|||
schedulerEvent.setName(values.get("name")); |
|||
schedulerEvent.setType(values.get("type")); |
|||
schedulerEvent.setTenantId(tenantId(values.get("tenant_id"))); |
|||
schedulerEvent.setConfiguration(toJsonNode(values.get("configuration"))); |
|||
schedulerEvent.setSchedule(toJsonNode(values.get("schedule"))); |
|||
schedulerEvent.setOriginatorId(entityId(values.get("originator_type"), values.get("originator_id"))); |
|||
schedulerEvent.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
|
|||
edqsService.onUpdate(schedulerEvent.getTenantId(), schedulerEvent.getId(), schedulerEvent); |
|||
}); |
|||
} |
|||
|
|||
private void loadRoles() throws Exception { |
|||
load("role.csv", (values) -> { |
|||
Role role = new Role(); |
|||
role.setId(new RoleId(uuid(values.get("id")))); |
|||
role.setCreatedTime(parseLong(values.get("created_time"))); |
|||
role.setName(values.get("name")); |
|||
role.setType(RoleType.valueOf(values.get("type"))); |
|||
role.setTenantId(tenantId(values.get("tenant_id"))); |
|||
role.setAdditionalInfo(toJsonNode(values.get("additional_info"))); |
|||
|
|||
edqsService.onUpdate(role.getTenantId(), role.getId(), role); |
|||
}); |
|||
} |
|||
|
|||
private void loadApiUsageStates() throws Exception { |
|||
load("api_usage_state.csv", (values) -> { |
|||
ApiUsageState apiUsageState = new ApiUsageState(); |
|||
apiUsageState.setId(new ApiUsageStateId(uuid(values.get("id")))); |
|||
apiUsageState.setCreatedTime(parseLong(values.get("created_time"))); |
|||
apiUsageState.setEntityId(entityId(values.get("entity_type"), values.get("entity_id"))); |
|||
apiUsageState.setTenantId(tenantId(values.get("tenant_id"))); |
|||
|
|||
edqsService.onUpdate(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState); |
|||
}); |
|||
} |
|||
|
|||
private void loadDeviceProfile() throws Exception { |
|||
load("device_profile.csv", (values) -> { |
|||
DeviceProfile deviceProfile = new DeviceProfile(); |
|||
deviceProfile.setId(new DeviceProfileId(uuid(values.get("id")))); |
|||
deviceProfile.setCreatedTime(parseLong(values.get("created_time"))); |
|||
deviceProfile.setName(values.get("name")); |
|||
deviceProfile.setType(DeviceProfileType.valueOf(values.get("type"))); |
|||
deviceProfile.setTenantId(tenantId(values.get("tenant_id"))); |
|||
|
|||
edqsService.onUpdate(deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile); |
|||
}); |
|||
} |
|||
|
|||
private void loadAssetProfile() throws Exception { |
|||
load("asset_profile.csv", (values) -> { |
|||
AssetProfile assetProfile = new AssetProfile(); |
|||
assetProfile.setId(new AssetProfileId(uuid(values.get("id")))); |
|||
assetProfile.setCreatedTime(parseLong(values.get("created_time"))); |
|||
assetProfile.setName(values.get("name")); |
|||
assetProfile.setTenantId(tenantId(values.get("tenant_id"))); |
|||
|
|||
edqsService.onUpdate(assetProfile.getTenantId(), assetProfile.getId(), assetProfile); |
|||
}); |
|||
} |
|||
|
|||
private void loadAttributes() throws Exception { |
|||
load("attribute.csv", (values) -> { |
|||
EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("entity_type"), values.get("entity_id")); |
|||
long ts = parseLong(values.get("last_update_ts")); |
|||
AttributeScope scope = AttributeScope.valueOf(values.get("attribute_type")); |
|||
String key = values.get("attribute_key"); |
|||
KvEntry kvEntry; |
|||
if (StringUtils.isNotEmpty(values.get("bool_v"))) { |
|||
kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v"))); |
|||
} else if (StringUtils.isNotEmpty(values.get("str_v"))) { |
|||
kvEntry = new StringDataEntry(key, values.get("str_v")); |
|||
} else if (StringUtils.isNotEmpty(values.get("long_v"))) { |
|||
kvEntry = new LongDataEntry(key, parseLong(values.get("long_v"))); |
|||
} else if (StringUtils.isNotEmpty(values.get("dbl_v"))) { |
|||
kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v"))); |
|||
} else if (StringUtils.isNotEmpty(values.get("json_v"))) { |
|||
kvEntry = new JsonDataEntry(key, values.get("json_v")); |
|||
} else { |
|||
kvEntry = new StringDataEntry(key, ""); |
|||
} |
|||
AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(ts, kvEntry); |
|||
AttributeKv attributeKv = new AttributeKv(entityId, scope, attributeKvEntry, 0); |
|||
edqsService.onUpdate(MAIN, ObjectType.ATTRIBUTE_KV, attributeKv); |
|||
}); |
|||
} |
|||
|
|||
private void loadTs() throws Exception { |
|||
load("ts_kv.csv", (values) -> { |
|||
var entityTypeStr = values.get("find_entity_type"); |
|||
if (StringUtils.isEmpty(entityTypeStr)) { |
|||
return; |
|||
} |
|||
EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("find_entity_type"), values.get("entity_id")); |
|||
long ts = parseLong(values.get("ts")); |
|||
String key = values.get("key"); |
|||
KvEntry kvEntry; |
|||
if (StringUtils.isNotEmpty(values.get("bool_v"))) { |
|||
kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v"))); |
|||
} else if (StringUtils.isNotEmpty(values.get("str_v"))) { |
|||
kvEntry = new StringDataEntry(key, values.get("str_v")); |
|||
} else if (StringUtils.isNotEmpty(values.get("long_v"))) { |
|||
kvEntry = new LongDataEntry(key, parseLong(values.get("long_v"))); |
|||
} else if (StringUtils.isNotEmpty(values.get("dbl_v"))) { |
|||
kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v"))); |
|||
} else if (StringUtils.isNotEmpty(values.get("json_v"))) { |
|||
kvEntry = new JsonDataEntry(key, values.get("json_v")); |
|||
} else { |
|||
kvEntry = new StringDataEntry(key, ""); |
|||
} |
|||
BasicTsKvEntry tsKvEntry = new BasicTsKvEntry(ts, kvEntry); |
|||
edqsService.onUpdate(MAIN, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, 0L)); |
|||
}); |
|||
} |
|||
|
|||
private void load(String file, Consumer<Map<String, String>> function) throws Exception { |
|||
Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("loader-" + file)).submit(() -> { |
|||
try { |
|||
long ts = System.currentTimeMillis(); |
|||
CsvSchema schema = CsvSchema.emptySchema().withHeader().withColumnSeparator('|'); |
|||
CsvMapper mapper = new CsvMapper(); |
|||
MappingIterator<Map<String, String>> it = mapper |
|||
.readerFor(Map.class) |
|||
.with(schema) |
|||
.readValues(new FileReader(folder + "/" + file)); |
|||
|
|||
int success = 0; |
|||
int failure = 0; |
|||
while (it.hasNextValue()) { |
|||
Map<String, String> row = it.nextValue(); |
|||
try { |
|||
function.accept(row); |
|||
success++; |
|||
if (success % 1000 == 0) { |
|||
log.info("Loaded [{}] from [{}]", success, file); |
|||
} |
|||
} catch (Exception e) { |
|||
log.error("Failed to parse str: [{}]", row, e); |
|||
failure++; |
|||
} |
|||
} |
|||
log.info("Loaded [{}] from [{}] in {}ms. Failures {}", success, file, (System.currentTimeMillis() - ts), failure); |
|||
} catch (Throwable t) { |
|||
log.error("Failed to load data from [{}]", file, t); |
|||
} |
|||
}); |
|||
} |
|||
|
|||
private static TenantId tenantId(String id) { |
|||
return TenantId.fromUUID(UUID.fromString(id)); |
|||
} |
|||
|
|||
private static CustomerId customerId(String id) { |
|||
var c = new CustomerId(UUID.fromString(id)); |
|||
return c.isNullUid() ? null : c; |
|||
} |
|||
|
|||
private static EntityId entityId(String type, String id) { |
|||
return EntityIdFactory.getByTypeAndId(type, id); |
|||
} |
|||
|
|||
private static UUID uuid(String id) { |
|||
return UUID.fromString(id); |
|||
} |
|||
|
|||
private static long parseLong(String time) { |
|||
return Long.parseLong(time); |
|||
} |
|||
|
|||
private static boolean parseBoolean(String value) { |
|||
return Boolean.parseBoolean(value); |
|||
} |
|||
|
|||
} |
|||
Loading…
Reference in new issue