From 5b8c8d1aedeb367a65c227db42e6d71d96ca2aa2 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 4 Apr 2025 16:11:46 +0300 Subject: [PATCH 01/15] UI: Show error for bad xml scada symbols --- .../scada-symbol/scada-symbol.component.ts | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts index 955ab16f97..caaf0edc01 100644 --- a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts +++ b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts @@ -79,6 +79,7 @@ import { SaveWidgetTypeAsDialogResult } from '@home/pages/widget/save-widget-type-as-dialog.component'; import { WidgetService } from '@core/http/widget.service'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; @Component({ selector: 'tb-scada-symbol', @@ -214,31 +215,35 @@ export class ScadaSymbolComponent extends PageComponent } const metadata: ScadaSymbolMetadata = this.scadaSymbolFormGroup.get('metadata').value; const scadaSymbolContent = this.prepareScadaSymbolContent(metadata); - const file = createFileFromContent(scadaSymbolContent, this.symbolData.imageResource.fileName, - this.symbolData.imageResource.descriptor.mediaType); - const type = imageResourceType(this.symbolData.imageResource); - let imageInfoObservable = - this.imageService.updateImage(type, this.symbolData.imageResource.resourceKey, file); - if (metadata.title !== this.symbolData.imageResource.title) { - imageInfoObservable = imageInfoObservable.pipe( - switchMap(imageInfo => { - imageInfo.title = metadata.title; - return this.imageService.updateImageInfo(imageInfo); - }) - ); - } - imageInfoObservable.pipe( - switchMap(imageInfo => this.imageService.getImageString( + if (scadaSymbolContent.includes('parsererror')) { + this.store.dispatch(new ActionNotificationShow({ message: scadaSymbolContent, type: 'error' })); + } else { + const file = createFileFromContent(scadaSymbolContent, this.symbolData.imageResource.fileName, + this.symbolData.imageResource.descriptor.mediaType); + const type = imageResourceType(this.symbolData.imageResource); + let imageInfoObservable = + this.imageService.updateImage(type, this.symbolData.imageResource.resourceKey, file); + if (metadata.title !== this.symbolData.imageResource.title) { + imageInfoObservable = imageInfoObservable.pipe( + switchMap(imageInfo => { + imageInfo.title = metadata.title; + return this.imageService.updateImageInfo(imageInfo); + }) + ); + } + imageInfoObservable.pipe( + switchMap(imageInfo => this.imageService.getImageString( `${IMAGES_URL_PREFIX}/${type}/${encodeURIComponent(imageInfo.resourceKey)}`).pipe( - map(content => ({ - imageResource: imageInfo, - scadaSymbolContent: content - })) + map(content => ({ + imageResource: imageInfo, + scadaSymbolContent: content + })) )) - ).subscribe(data => { - this.init(data); - this.updateBreadcrumbs.emit(); - }); + ).subscribe(data => { + this.init(data); + this.updateBreadcrumbs.emit(); + }); + } } } From ac3b389203abfeafbbb2959dcd8e8de71a13fc0c Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 10 Apr 2025 10:51:11 +0300 Subject: [PATCH 02/15] UI: Add Echart in modules map --- ui-ngx/src/app/modules/common/modules-map.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui-ngx/src/app/modules/common/modules-map.ts b/ui-ngx/src/app/modules/common/modules-map.ts index 70432e8cdf..530a41f006 100644 --- a/ui-ngx/src/app/modules/common/modules-map.ts +++ b/ui-ngx/src/app/modules/common/modules-map.ts @@ -76,6 +76,8 @@ import * as TranslateCore from '@ngx-translate/core'; import * as MatDateTimePicker from '@mat-datetimepicker/core'; import _moment from 'moment'; import * as tslib from 'tslib'; +import * as Echarts from 'echarts'; +import * as EchartsCore from 'echarts/core'; import * as TbCore from '@core/public-api'; import * as TbShared from '@shared/public-api'; @@ -405,6 +407,8 @@ class ModulesMap implements IModulesMap { '@mat-datetimepicker/core': MatDateTimePicker, moment: _moment, tslib, + 'echarts': Echarts, + 'echarts/core': EchartsCore, '@core/public-api': TbCore, '@shared/public-api': TbShared, From ced71331efa6e3e840b557a0b555e1d786ac724c Mon Sep 17 00:00:00 2001 From: deaflynx Date: Tue, 1 Apr 2025 12:04:26 +0300 Subject: [PATCH 03/15] UI: JavaScript library (Module type) minor style enhancement: remove redundant space before required symbol, decrease help popover width (prod-5636). --- .../home/pages/admin/resource/js-resource.component.html | 3 +-- ui-ngx/src/app/shared/components/js-func.component.scss | 4 ++++ .../src/assets/help/en_US/resource/js-resource-module_fn.md | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html b/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html index 6287d27f2c..15d899df6b 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html @@ -91,8 +91,7 @@ + {{ 'javascript.module-script' | translate }} +that are exported for use in other parts of the application. ##### Examples You can declare variables: From 713050937c50151eb5c87bb65426f2071326835f Mon Sep 17 00:00:00 2001 From: deaflynx Date: Thu, 10 Apr 2025 10:53:09 +0300 Subject: [PATCH 04/15] UI: JavaScript resource help popup too wide width issue: use [helpPopupStyle]. --- .../home/pages/admin/resource/js-resource.component.html | 1 + ui-ngx/src/assets/help/en_US/resource/js-resource-module_fn.md | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html b/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html index 15d899df6b..62036a8ea1 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html @@ -80,6 +80,7 @@ (fileNameChanged)="entityForm?.get('fileName').patchValue($event)"> -that are exported for use in other parts of the application. +These modules can contain any JavaScript code, facilitating the reuse of specific logic. This includes variables or functions that are exported for use in other parts of the application. ##### Examples You can declare variables: From 59fa19d1c861d10aca3a44073a1786c10aab0d9c Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Apr 2025 11:40:19 +0300 Subject: [PATCH 05/15] Change EDQS consumers poll interval from 125 ms to 25 ms --- application/src/main/resources/thingsboard.yml | 2 +- .../main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java | 2 +- edqs/src/main/resources/edqs.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index b4d178a737..42f5331e21 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1773,7 +1773,7 @@ queue: # EDQS responses topic responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" # Poll interval for EDQS topics - poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:25}" # Maximum amount of pending requests to EDQS max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" # Maximum timeout for requests to EDQS diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java index 401b451f59..3c927f135b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java @@ -38,7 +38,7 @@ public class EdqsConfig { private String requestsTopic; @Value("${queue.edqs.responses_topic:edqs.responses}") private String responsesTopic; - @Value("${queue.edqs.poll_interval:125}") + @Value("${queue.edqs.poll_interval:25}") private long pollInterval; @Value("${queue.edqs.max_pending_requests:10000}") private int maxPendingRequests; diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 2c0e0b8c5d..8ea40c03c8 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -66,7 +66,7 @@ queue: # EDQS responses topic responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" # Poll interval for EDQS topics - poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:25}" # Maximum amount of pending requests to EDQS max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" # Maximum timeout for requests to EDQS From d1f0ec5033774f02b9cf094d3eb42cb5f8bd1e79 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Apr 2025 11:41:19 +0300 Subject: [PATCH 06/15] Additionally check for string compression threshold on EDQS side --- .../server/edqs/util/EdqsConverter.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java index 167037b889..18e2f521e3 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -171,11 +171,21 @@ public class EdqsConverter { } else if (proto.hasDoubleV()) { return new DoubleDataPoint(ts, proto.getDoubleV()); } else if (proto.hasStringV()) { - return new StringDataPoint(ts, proto.getStringV()); + String stringV = proto.getStringV(); + if (stringV.length() < stringCompressionLengthThreshold) { + return new StringDataPoint(ts, stringV); + } else { + return new CompressedStringDataPoint(ts, compress(stringV)); + } } else if (proto.hasCompressedStringV()) { return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray()); } else if (proto.hasJsonV()) { - return new JsonDataPoint(ts, proto.getJsonV()); + String jsonV = proto.getJsonV(); + if (jsonV.length() < stringCompressionLengthThreshold) { + return new JsonDataPoint(ts, jsonV); + } else { + return new CompressedJsonDataPoint(ts, compress(jsonV)); + } } else if (proto.hasCompressedJsonV()) { return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray()); } else { From b806a41e62d473df72a1e2c04b1536d29690a717 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Apr 2025 14:02:56 +0300 Subject: [PATCH 07/15] More stats for EDQS --- .../server/edqs/data/BaseEntityData.java | 2 +- .../edqs/data/dp/CompressedJsonDataPoint.java | 6 +- .../data/dp/CompressedStringDataPoint.java | 10 +- .../edqs/repo/DefaultEdqsRepository.java | 9 +- .../server/edqs/repo/TenantRepo.java | 2 - .../edqs/stats/DefaultEdqsStatsService.java | 93 ++++++++++++++----- .../server/edqs/util/EdqsConverter.java | 29 ++++-- .../common/stats/DefaultStatsFactory.java | 11 +++ .../common/stats/DummyEdqsStatsService.java | 16 +++- .../server/common/stats/EdqsStatsService.java | 12 ++- .../server/common/stats/StatsFactory.java | 6 ++ .../thingsboard/common/util/TbBytePool.java | 6 +- .../thingsboard/common/util/TbStringPool.java | 6 +- .../server/dao/entity/BaseEntityService.java | 9 +- edqs/src/main/resources/edqs.yml | 2 + 15 files changed, 158 insertions(+), 61 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java index 33f32b9781..21b9ff0c4f 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java @@ -125,7 +125,7 @@ public abstract class BaseEntityData implements EntityDa return switch (key) { case "createdTime" -> new LongDataPoint(System.currentTimeMillis(), fields.getCreatedTime()); case "edgeTemplate" -> new BoolDataPoint(System.currentTimeMillis(), fields.isEdgeTemplate()); - case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx)); + case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx), false); default -> new StringDataPoint(System.currentTimeMillis(), getField(key), false); }; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java index bce9d86875..c05a724e79 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java @@ -17,10 +17,12 @@ package org.thingsboard.server.edqs.data.dp; import org.thingsboard.server.common.data.kv.DataType; +import java.util.function.Function; + public class CompressedJsonDataPoint extends CompressedStringDataPoint { - public CompressedJsonDataPoint(long ts, byte[] compressedValue) { - super(ts, compressedValue); + public CompressedJsonDataPoint(long ts, byte[] compressedValue, Function uncompressor) { + super(ts, compressedValue, uncompressor); } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java index cf4267e443..45db6bf72a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java @@ -19,17 +19,21 @@ import lombok.Getter; import lombok.SneakyThrows; import org.thingsboard.common.util.TbBytePool; import org.thingsboard.server.common.data.kv.DataType; -import org.xerial.snappy.Snappy; + +import java.util.function.Function; public class CompressedStringDataPoint extends AbstractDataPoint { @Getter private final byte[] compressedValue; + protected final Function uncompressor; + @SneakyThrows - public CompressedStringDataPoint(long ts, byte[] compressedValue) { + public CompressedStringDataPoint(long ts, byte[] compressedValue, Function uncompressor) { super(ts); this.compressedValue = TbBytePool.intern(compressedValue); + this.uncompressor = uncompressor; } @Override @@ -40,7 +44,7 @@ public class CompressedStringDataPoint extends AbstractDataPoint { @SneakyThrows @Override public String getStr() { - return Snappy.uncompressString(compressedValue); + return uncompressor.apply(compressedValue); } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java index 215c64194f..88e4cc63d2 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edqs.repo; import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; @@ -40,6 +41,7 @@ import java.util.function.Predicate; @Slf4j public class DefaultEdqsRepository implements EdqsRepository { + @Getter private final static ConcurrentMap repos = new ConcurrentHashMap<>(); private final EdqsStatsService statsService; @@ -52,6 +54,7 @@ public class DefaultEdqsRepository implements EdqsRepository { if (event.getEventType() == EdqsEventType.DELETED && event.getObjectType() == ObjectType.TENANT) { log.info("Tenant {} deleted", event.getTenantId()); repos.remove(event.getTenantId()); + statsService.reportRemoved(ObjectType.TENANT); } else { get(event.getTenantId()).processEvent(event); } @@ -61,8 +64,7 @@ public class DefaultEdqsRepository implements EdqsRepository { public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck) { long startNs = System.nanoTime(); long result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); - double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; - log.info("countEntitiesByQuery done in {} ms", timingMs); + statsService.reportEdqsCountQuery(tenantId, query, System.nanoTime() - startNs); return result; } @@ -71,8 +73,7 @@ public class DefaultEdqsRepository implements EdqsRepository { EntityDataQuery query, boolean ignorePermissionCheck) { long startNs = System.nanoTime(); var result = get(tenantId).findEntityDataByQuery(customerId, query, ignorePermissionCheck); - double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; - log.info("findEntityDataByQuery done in {} ms", timingMs); + statsService.reportEdqsDataQuery(tenantId, query, System.nanoTime() - startNs); return result; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index a47559d6d8..ffba7583c9 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -332,7 +332,6 @@ public class TenantRepo { public PageData findEntityDataByQuery(CustomerId customerId, EntityDataQuery oldQuery, boolean ignorePermissionCheck) { EdqsDataQuery query = RepositoryUtils.toNewQuery(oldQuery); - log.info("[{}][{}] findEntityDataByQuery: {}", tenantId, customerId, query); QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck); EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); return sortAndConvert(query, queryProcessor.processQuery(), ctx); @@ -340,7 +339,6 @@ public class TenantRepo { public long countEntitiesByQuery(CustomerId customerId, EntityCountQuery oldQuery, boolean ignorePermissionCheck) { EdqsQuery query = RepositoryUtils.toNewQuery(oldQuery); - log.info("[{}][{}] countEntitiesByQuery: {}", tenantId, customerId, query); QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck); EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); return queryProcessor.count(); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java index 3767d0f60b..46fb153771 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java @@ -15,78 +15,123 @@ */ package org.thingsboard.server.edqs.stats; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.TbBytePool; +import org.thingsboard.common.util.TbStringPool; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.stats.EdqsStatsService; +import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsTimer; import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.edqs.repo.DefaultEdqsRepository; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @Service @Slf4j +@RequiredArgsConstructor @ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && '${queue.edqs.stats.enabled:true}' == 'true'") public class DefaultEdqsStatsService implements EdqsStatsService { private final StatsFactory statsFactory; - @Value("${queue.edqs.stats.slow_query_threshold:3000}") + @Value("${queue.edqs.stats.slow_query_threshold}") private int slowQueryThreshold; - private final ConcurrentHashMap objectCounters = new ConcurrentHashMap<>(); - private final StatsTimer dataQueryTimer; - private final StatsTimer countQueryTimer; + private final ConcurrentMap objectCounters = new ConcurrentHashMap<>(); + private final ConcurrentMap timers = new ConcurrentHashMap<>(); + private final ConcurrentMap counters = new ConcurrentHashMap<>(); + private final ConcurrentMap gauges = new ConcurrentHashMap<>(); - private DefaultEdqsStatsService(StatsFactory statsFactory) { - this.statsFactory = statsFactory; - dataQueryTimer = statsFactory.createTimer(StatsType.EDQS, "entityDataQueryTimer"); - countQueryTimer = statsFactory.createTimer(StatsType.EDQS, "entityCountQueryTimer"); + @PostConstruct + private void init() { + statsFactory.createGauge(StatsType.EDQS, "stringPoolSize", TbStringPool.getPool(), Map::size); + statsFactory.createGauge(StatsType.EDQS, "bytePoolSize", TbBytePool.getPool(), Map::size); + statsFactory.createGauge(StatsType.EDQS, "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size); } @Override public void reportAdded(ObjectType objectType) { - getObjectCounter(objectType).incrementAndGet(); + getObjectGauge(objectType).incrementAndGet(); } @Override public void reportRemoved(ObjectType objectType) { - getObjectCounter(objectType).decrementAndGet(); + getObjectGauge(objectType).decrementAndGet(); } @Override - public void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) { - double timingMs = timingNanos / 1000_000.0; - if (timingMs < slowQueryThreshold) { - log.debug("[{}] Executed data query in {} ms: {}", tenantId, timingMs, query); - } else { - log.warn("[{}] Executed slow data query in {} ms: {}", tenantId, timingMs, query); - } - dataQueryTimer.record(timingNanos, TimeUnit.NANOSECONDS); + public void reportEntityDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) { + checkTiming(tenantId, query, timingNanos); + getTimer("entityDataQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void reportEntityCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) { + checkTiming(tenantId, query, timingNanos); + getTimer("entityCountQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void reportEdqsDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) { + checkTiming(tenantId, query, timingNanos); + getTimer("edqsDataQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void reportEdqsCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) { + checkTiming(tenantId, query, timingNanos); + getTimer("edqsCountQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void reportStringCompressed() { + getCounter("stringsCompressed").increment(); } @Override - public void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) { + public void reportStringUncompressed() { + getCounter("stringsUncompressed").increment(); + } + + private void checkTiming(TenantId tenantId, EntityCountQuery query, long timingNanos) { double timingMs = timingNanos / 1000_000.0; + String queryType = query instanceof EntityDataQuery ? "data" : "count"; if (timingMs < slowQueryThreshold) { - log.debug("[{}] Executed count query in {} ms: {}", tenantId, timingMs, query); + log.debug("[{}] Executed " + queryType + " query in {} ms: {}", tenantId, timingMs, query); } else { - log.warn("[{}] Executed slow count query in {} ms: {}", tenantId, timingMs, query); + log.warn("[{}] Executed slow " + queryType + " query in {} ms: {}", tenantId, timingMs, query); } - countQueryTimer.record(timingNanos, TimeUnit.NANOSECONDS); } - private AtomicInteger getObjectCounter(ObjectType objectType) { + private StatsTimer getTimer(String name) { + return timers.computeIfAbsent(name, __ -> statsFactory.createTimer(StatsType.EDQS, name)); + } + + private StatsCounter getCounter(String name) { + return counters.computeIfAbsent(name, __ -> statsFactory.createStatsCounter(StatsType.EDQS.getName(), name)); + } + + private AtomicInteger getGauge(String name) { + return gauges.computeIfAbsent(name, __ -> statsFactory.createGauge(StatsType.EDQS, name, new AtomicInteger())); + } + + private AtomicInteger getObjectGauge(ObjectType objectType) { return objectCounters.computeIfAbsent(objectType, type -> - statsFactory.createGauge("edqsObjectsCount", new AtomicInteger(), "objectType", type.name())); + statsFactory.createGauge(StatsType.EDQS, "objectsCount", new AtomicInteger(), "objectType", type.name())); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java index 18e2f521e3..9a84436f36 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.stats.EdqsStatsService; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.edqs.data.dp.BoolDataPoint; import org.thingsboard.server.edqs.data.dp.CompressedJsonDataPoint; @@ -56,14 +57,18 @@ import org.thingsboard.server.gen.transport.TransportProtos.DataPointProto; import org.xerial.snappy.Snappy; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.UUID; @Service +@RequiredArgsConstructor @Slf4j public class EdqsConverter { + private final EdqsStatsService edqsStatsService; + @Value("${queue.edqs.string_compression_length_threshold:512}") private int stringCompressionLengthThreshold; @@ -175,32 +180,40 @@ public class EdqsConverter { if (stringV.length() < stringCompressionLengthThreshold) { return new StringDataPoint(ts, stringV); } else { - return new CompressedStringDataPoint(ts, compress(stringV)); + return new CompressedStringDataPoint(ts, compress(stringV), this::uncompress); } } else if (proto.hasCompressedStringV()) { - return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray()); + return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray(), this::uncompress); } else if (proto.hasJsonV()) { String jsonV = proto.getJsonV(); if (jsonV.length() < stringCompressionLengthThreshold) { return new JsonDataPoint(ts, jsonV); } else { - return new CompressedJsonDataPoint(ts, compress(jsonV)); + return new CompressedJsonDataPoint(ts, compress(jsonV), this::uncompress); } } else if (proto.hasCompressedJsonV()) { - return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray()); + return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray(), this::uncompress); } else { throw new IllegalArgumentException("Unsupported data point proto: " + proto); } } @SneakyThrows - private static byte[] compress(String value) { - byte[] compressed = Snappy.compress(value); - // TODO: limit the size - log.debug("Compressed {} bytes to {} bytes", value.length(), compressed.length); + private byte[] compress(String value) { + byte[] compressed = Snappy.compress(value, StandardCharsets.UTF_8); + log.debug("Compressed {} chars to {} bytes", value.length(), compressed.length); + edqsStatsService.reportStringCompressed(); return compressed; } + @SneakyThrows + private String uncompress(byte[] compressed) { + String value = Snappy.uncompressString(compressed, StandardCharsets.UTF_8); + log.debug("Uncompressed {} bytes to {} chars", compressed.length, value.length()); + edqsStatsService.reportStringUncompressed(); + return value; + } + public static Entity toEntity(EntityType entityType, Object entity) { Entity edqsEntity = new Entity(); edqsEntity.setType(entityType); diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index 8e2291270d..15c155ecfd 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.StringUtils; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.ToDoubleFunction; @Service public class DefaultStatsFactory implements StatsFactory { @@ -86,6 +87,16 @@ public class DefaultStatsFactory implements StatsFactory { return meterRegistry.gauge(key, Tags.of(tags), number); } + @Override + public T createGauge(StatsType statsType, String name, T number, String... tags) { + return createGauge(statsType.getName(), number, getTags(name, tags)); + } + + @Override + public void createGauge(StatsType statsType, String name, S stateObject, ToDoubleFunction numberProvider, String... tags) { + meterRegistry.gauge(statsType.getName(), Tags.of(getTags(name, tags)), stateObject, numberProvider); + } + @Override public MessagesStats createMessagesStats(String key) { StatsCounter totalCounter = createStatsCounter(key, TOTAL_MSGS); diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java index df78e5fc89..3db1c6c790 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java @@ -33,9 +33,21 @@ public class DummyEdqsStatsService implements EdqsStatsService { public void reportRemoved(ObjectType objectType) {} @Override - public void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {} + public void reportEntityDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {} @Override - public void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {} + public void reportEntityCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {} + + @Override + public void reportEdqsDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {} + + @Override + public void reportEdqsCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {} + + @Override + public void reportStringCompressed() {} + + @Override + public void reportStringUncompressed() {} } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java index 106e43e913..3a6eeb26b8 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java @@ -26,8 +26,16 @@ public interface EdqsStatsService { void reportRemoved(ObjectType objectType); - void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos); + void reportEntityDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos); - void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos); + void reportEntityCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos); + + void reportEdqsDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos); + + void reportEdqsCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos); + + void reportStringCompressed(); + + void reportStringUncompressed(); } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java index bd46c09285..e438af3ad7 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java @@ -17,6 +17,8 @@ package org.thingsboard.server.common.stats; import io.micrometer.core.instrument.Timer; +import java.util.function.ToDoubleFunction; + public interface StatsFactory { StatsCounter createStatsCounter(String key, String statsName, String... otherTags); @@ -25,6 +27,10 @@ public interface StatsFactory { T createGauge(String key, T number, String... tags); + T createGauge(StatsType statsType, String name, T number, String... tags); + + void createGauge(StatsType statsType, String name, S stateObject, ToDoubleFunction numberProvider, String... tags); + MessagesStats createMessagesStats(String key); Timer createTimer(String key, String... tags); diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java b/common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java index fe16a14e7c..1aadb49129 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java @@ -16,12 +16,14 @@ package org.thingsboard.common.util; import com.google.common.hash.Hashing; +import lombok.Getter; import org.springframework.util.ConcurrentReferenceHashMap; import java.util.concurrent.ConcurrentMap; public class TbBytePool { + @Getter private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); public static byte[] intern(byte[] data) { @@ -32,8 +34,4 @@ public class TbBytePool { return pool.computeIfAbsent(checksum, c -> data); } - public static int size(){ - return pool.size(); - } - } diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java b/common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java index 38c010fbd3..167bf1bbed 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java @@ -15,12 +15,14 @@ */ package org.thingsboard.common.util; +import lombok.Getter; import org.springframework.util.ConcurrentReferenceHashMap; import java.util.concurrent.ConcurrentMap; public class TbStringPool { + @Getter private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); public static String intern(String data) { @@ -30,8 +32,4 @@ public class TbStringPool { return pool.computeIfAbsent(data, str -> str); } - public static int size(){ - return pool.size(); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 4df33779ee..ee9320cce3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasEmail; @@ -101,7 +100,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); - TbStopWatch stopWatch = TbStopWatch.create(); + long startNs = System.nanoTime(); Long result; if (edqsApiService.isEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { EdqsRequest request = EdqsRequest.builder() @@ -112,7 +111,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe } else { result = entityQueryDao.countEntitiesByQuery(tenantId, customerId, query); } - edqsStatsService.reportCountQuery(tenantId, query, stopWatch.stopAndGetTotalTimeNanos()); + edqsStatsService.reportEntityCountQuery(tenantId, query, System.nanoTime() - startNs); return result; } @@ -123,7 +122,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); - TbStopWatch stopWatch = TbStopWatch.create(); + long startNs = System.nanoTime(); PageData result; if (edqsApiService.isEnabled() && validForEdqs(query)) { EdqsRequest request = EdqsRequest.builder() @@ -146,7 +145,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe } } } - edqsStatsService.reportDataQuery(tenantId, query, stopWatch.stopAndGetTotalTimeNanos()); + edqsStatsService.reportEntityDataQuery(tenantId, query, System.nanoTime() - startNs); return result; } diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 8ea40c03c8..1cc32a4230 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -76,6 +76,8 @@ queue: stats: # Enable/disable statistics for EDQS enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Threshold for slow queries to log, in milliseconds + slow_query_threshold: "${TB_EDQS_SLOW_QUERY_THRESHOLD_MS:200}" kafka: # Kafka Bootstrap nodes in "host:port" format From 70906101683eddb1afa178cbde0c17c543b657a9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Apr 2025 14:54:42 +0300 Subject: [PATCH 08/15] Fix EDQS stats collector names --- .../stats/HousekeeperStatsService.java | 2 +- .../edqs/stats/DefaultEdqsStatsService.java | 15 +++++++-------- .../server/common/stats/DefaultStatsFactory.java | 12 ++++++------ .../server/common/stats/StatsFactory.java | 6 +++--- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java index 5da5fa2be0..5946b84d09 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -106,7 +106,7 @@ public class HousekeeperStatsService { this.failedProcessingCounter = register("failedProcessing", statsFactory); this.reprocessedCounter = register("reprocessed", statsFactory); this.failedReprocessingCounter = register("failedReprocessing", statsFactory); - this.processingTimer = statsFactory.createTimer(StatsType.HOUSEKEEPER, "processingTime", "taskType", taskType.name()); + this.processingTimer = statsFactory.createStatsTimer(StatsType.HOUSEKEEPER.getName(), "processingTime", "taskType", taskType.name()); } private StatsCounter register(String statsName, StatsFactory statsFactory) { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java index 46fb153771..4853920c44 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java @@ -31,7 +31,6 @@ import org.thingsboard.server.common.stats.EdqsStatsService; import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsTimer; -import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.edqs.repo.DefaultEdqsRepository; import java.util.Map; @@ -58,9 +57,9 @@ public class DefaultEdqsStatsService implements EdqsStatsService { @PostConstruct private void init() { - statsFactory.createGauge(StatsType.EDQS, "stringPoolSize", TbStringPool.getPool(), Map::size); - statsFactory.createGauge(StatsType.EDQS, "bytePoolSize", TbBytePool.getPool(), Map::size); - statsFactory.createGauge(StatsType.EDQS, "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size); + statsFactory.createGauge("edqsGauges", "stringPoolSize", TbStringPool.getPool(), Map::size); + statsFactory.createGauge("edqsGauges", "bytePoolSize", TbBytePool.getPool(), Map::size); + statsFactory.createGauge("edqsGauges", "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size); } @Override @@ -118,20 +117,20 @@ public class DefaultEdqsStatsService implements EdqsStatsService { } private StatsTimer getTimer(String name) { - return timers.computeIfAbsent(name, __ -> statsFactory.createTimer(StatsType.EDQS, name)); + return timers.computeIfAbsent(name, __ -> statsFactory.createStatsTimer("edqsTimers", name)); } private StatsCounter getCounter(String name) { - return counters.computeIfAbsent(name, __ -> statsFactory.createStatsCounter(StatsType.EDQS.getName(), name)); + return counters.computeIfAbsent(name, __ -> statsFactory.createStatsCounter("edqsCounters", name)); } private AtomicInteger getGauge(String name) { - return gauges.computeIfAbsent(name, __ -> statsFactory.createGauge(StatsType.EDQS, name, new AtomicInteger())); + return gauges.computeIfAbsent(name, __ -> statsFactory.createGauge("edqsGauges", name, new AtomicInteger())); } private AtomicInteger getObjectGauge(ObjectType objectType) { return objectCounters.computeIfAbsent(objectType, type -> - statsFactory.createGauge(StatsType.EDQS, "objectsCount", new AtomicInteger(), "objectType", type.name())); + statsFactory.createGauge("edqsGauges", "objectsCount", new AtomicInteger(), "objectType", type.name())); } } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index 15c155ecfd..4d858be4bb 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -88,13 +88,13 @@ public class DefaultStatsFactory implements StatsFactory { } @Override - public T createGauge(StatsType statsType, String name, T number, String... tags) { - return createGauge(statsType.getName(), number, getTags(name, tags)); + public T createGauge(String type, String name, T number, String... tags) { + return createGauge(type, number, getTags(name, tags)); } @Override - public void createGauge(StatsType statsType, String name, S stateObject, ToDoubleFunction numberProvider, String... tags) { - meterRegistry.gauge(statsType.getName(), Tags.of(getTags(name, tags)), stateObject, numberProvider); + public void createGauge(String type, String name, S stateObject, ToDoubleFunction numberProvider, String... tags) { + meterRegistry.gauge(type, Tags.of(getTags(name, tags)), stateObject, numberProvider); } @Override @@ -117,8 +117,8 @@ public class DefaultStatsFactory implements StatsFactory { } @Override - public StatsTimer createTimer(StatsType type, String name, String... tags) { - return new StatsTimer(name, Timer.builder(type.getName()) + public StatsTimer createStatsTimer(String type, String name, String... tags) { + return new StatsTimer(name, Timer.builder(type) .tags(getTags(name, tags)) .register(meterRegistry)); } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java index e438af3ad7..291b95e25a 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java @@ -27,14 +27,14 @@ public interface StatsFactory { T createGauge(String key, T number, String... tags); - T createGauge(StatsType statsType, String name, T number, String... tags); + T createGauge(String type, String name, T number, String... tags); - void createGauge(StatsType statsType, String name, S stateObject, ToDoubleFunction numberProvider, String... tags); + void createGauge(String type, String name, S stateObject, ToDoubleFunction numberProvider, String... tags); MessagesStats createMessagesStats(String key); Timer createTimer(String key, String... tags); - StatsTimer createTimer(StatsType type, String name, String... tags); + StatsTimer createStatsTimer(String type, String name, String... tags); } From b854dcf0c7242560bbb87d50f3ef6ea6e12e5df1 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Thu, 10 Apr 2025 14:59:11 +0300 Subject: [PATCH 09/15] UI: Refactoring error msg --- .../components/widget/lib/scada/scada-symbol.models.ts | 2 +- .../home/pages/scada-symbol/scada-symbol.component.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts index e52c6f9f54..abe570fa3c 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts @@ -271,7 +271,7 @@ export const updateScadaSymbolMetadataInContent = (svgContent: string, metadata: const svgDoc = new DOMParser().parseFromString(svgContent, 'image/svg+xml'); const parsererror = svgDoc.getElementsByTagName('parsererror'); if (parsererror?.length) { - return parsererror[0].outerHTML; + throw Error(parsererror[0].textContent) } updateScadaSymbolMetadataInDom(svgDoc, metadata); return svgDoc.documentElement.outerHTML; diff --git a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts index caaf0edc01..c618380565 100644 --- a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts +++ b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts @@ -214,10 +214,8 @@ export class ScadaSymbolComponent extends PageComponent this.editObjectCallbacks.tagsUpdated(tags); } const metadata: ScadaSymbolMetadata = this.scadaSymbolFormGroup.get('metadata').value; - const scadaSymbolContent = this.prepareScadaSymbolContent(metadata); - if (scadaSymbolContent.includes('parsererror')) { - this.store.dispatch(new ActionNotificationShow({ message: scadaSymbolContent, type: 'error' })); - } else { + try { + const scadaSymbolContent = this.prepareScadaSymbolContent(metadata); const file = createFileFromContent(scadaSymbolContent, this.symbolData.imageResource.fileName, this.symbolData.imageResource.descriptor.mediaType); const type = imageResourceType(this.symbolData.imageResource); @@ -243,6 +241,8 @@ export class ScadaSymbolComponent extends PageComponent this.init(data); this.updateBreadcrumbs.emit(); }); + } catch (e) { + this.store.dispatch(new ActionNotificationShow({ message: e.message, type: 'error' })); } } } From 6af342090e577016145a3023c04f0dd9782a6e1f Mon Sep 17 00:00:00 2001 From: Ekaterina Chantsova Date: Thu, 10 Apr 2025 15:30:20 +0300 Subject: [PATCH 10/15] Timewindow: added timezone selection in widget settings --- .../widget/config/timewindow-config-panel.component.html | 1 + 1 file changed, 1 insertion(+) diff --git a/ui-ngx/src/app/modules/home/components/widget/config/timewindow-config-panel.component.html b/ui-ngx/src/app/modules/home/components/widget/config/timewindow-config-panel.component.html index ff0acd5fca..138dde324e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/timewindow-config-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/timewindow-config-panel.component.html @@ -33,6 +33,7 @@ noMargin isEdit="true" alwaysDisplayTypePrefix + timezone="true" [historyOnly]="onlyHistoryTimewindow" quickIntervalOnly="{{ widgetType === widgetTypes.latest }}" aggregation="{{ widgetType === widgetTypes.timeseries }}" From 2abebbd352f6354d3920e207d84c21a40563ad2f Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 10 Apr 2025 15:35:25 +0300 Subject: [PATCH 11/15] UI: Improved CSS styles for copy-code button in markdown --- .../device-check-connectivity-dialog.component.scss | 1 - .../src/app/shared/components/markdown.component.scss | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/device/device-check-connectivity-dialog.component.scss b/ui-ngx/src/app/modules/home/pages/device/device-check-connectivity-dialog.component.scss index 123bcfb708..eca728893c 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-check-connectivity-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/pages/device/device-check-connectivity-dialog.component.scss @@ -124,7 +124,6 @@ background: #F3F6FA; border-color: #305680; padding-right: 38px; - overflow: scroll; padding-bottom: 4px; min-height: 42px; scrollbar-width: thin; diff --git a/ui-ngx/src/app/shared/components/markdown.component.scss b/ui-ngx/src/app/shared/components/markdown.component.scss index 4989a77021..65ad64b189 100644 --- a/ui-ngx/src/app/shared/components/markdown.component.scss +++ b/ui-ngx/src/app/shared/components/markdown.component.scss @@ -270,7 +270,7 @@ outline: none; position: absolute; width: 206px; - height: 42px; + height: 32px; top: 0; right: 32px; background: 0 0; @@ -281,11 +281,11 @@ user-select: none; &.multiline { - right: 44px; + right: 38px; } p { - padding: 8px; + padding: 8px 8px 0; top: 1px; transition: .2s; color: #2a7dec; @@ -301,10 +301,10 @@ background-color: #fff; position: absolute; width: 38px; - height: 38px; + height: 28px; top: 3px; right: 3px; - padding: 10px; + padding: 10px 10px 0; img { position: initial; From 413cc12126796aea78fb6d98d6a7a2ff1113a6db Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 11 Apr 2025 09:34:44 +0300 Subject: [PATCH 12/15] UI: Handle error on apply preview upload and download --- .../scada-symbol/scada-symbol.component.ts | 96 ++++++++++--------- .../image/upload-image-dialog.component.html | 2 +- .../image/upload-image-dialog.component.ts | 63 +++++++----- 3 files changed, 90 insertions(+), 71 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts index c618380565..a2b47ee811 100644 --- a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts +++ b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts @@ -253,43 +253,47 @@ export class ScadaSymbolComponent extends PageComponent enterPreviewMode() { this.previewMetadata = this.scadaSymbolFormGroup.get('metadata').value; - this.symbolData.scadaSymbolContent = this.prepareScadaSymbolContent(this.previewMetadata); - this.previewScadaSymbolObjectSettings = { - behavior: {}, - properties: {} - }; - this.scadaPreviewFormGroup.patchValue({ - scadaSymbolObjectSettings: this.previewScadaSymbolObjectSettings - }, {emitEvent: false}); - this.scadaPreviewFormGroup.markAsPristine(); - const settings: ScadaSymbolWidgetSettings = {...scadaSymbolWidgetDefaultSettings, - ...{ + try { + this.symbolData.scadaSymbolContent = this.prepareScadaSymbolContent(this.previewMetadata); + this.previewScadaSymbolObjectSettings = { + behavior: {}, + properties: {} + }; + this.scadaPreviewFormGroup.patchValue({ + scadaSymbolObjectSettings: this.previewScadaSymbolObjectSettings + }, {emitEvent: false}); + this.scadaPreviewFormGroup.markAsPristine(); + const settings: ScadaSymbolWidgetSettings = {...scadaSymbolWidgetDefaultSettings, + ...{ simulated: true, scadaSymbolUrl: null, scadaSymbolContent: this.symbolData.scadaSymbolContent, scadaSymbolObjectSettings: this.previewScadaSymbolObjectSettings, padding: '0', background: colorBackground('rgba(0,0,0,0)') - } - }; - this.previewWidget = { - typeFullFqn: 'system.scada_symbol', - type: widgetType.rpc, - sizeX: this.previewMetadata.widgetSizeX || 3, - sizeY: this.previewMetadata.widgetSizeY || 3, - row: 0, - col: 0, - config: { - settings, - showTitle: false, - dropShadow: false, - padding: '0', - margin: '0', - backgroundColor: 'rgba(0,0,0,0)' - } - }; - this.previewWidgets = [this.previewWidget]; - this.previewMode = true; + } + }; + this.previewWidget = { + typeFullFqn: 'system.scada_symbol', + type: widgetType.rpc, + sizeX: this.previewMetadata.widgetSizeX || 3, + sizeY: this.previewMetadata.widgetSizeY || 3, + row: 0, + col: 0, + config: { + settings, + showTitle: false, + dropShadow: false, + padding: '0', + margin: '0', + backgroundColor: 'rgba(0,0,0,0)' + } + }; + this.previewWidgets = [this.previewWidget]; + this.previewMode = true; + } catch (e) { + this.store.dispatch(new ActionNotificationShow({ message: e.message, type: 'error' })); + } } exitPreviewMode() { @@ -379,19 +383,23 @@ export class ScadaSymbolComponent extends PageComponent metadata = parseScadaSymbolMetadataFromContent(this.origSymbolData.scadaSymbolContent); } const linkElement = document.createElement('a'); - const scadaSymbolContent = this.prepareScadaSymbolContent(metadata); - const blob = new Blob([scadaSymbolContent], { type: this.symbolData.imageResource.descriptor.mediaType }); - const url = URL.createObjectURL(blob); - linkElement.setAttribute('href', url); - linkElement.setAttribute('download', this.symbolData.imageResource.fileName); - const clickEvent = new MouseEvent('click', - { - view: window, - bubbles: true, - cancelable: false - } - ); - linkElement.dispatchEvent(clickEvent); + try { + const scadaSymbolContent = this.prepareScadaSymbolContent(metadata); + const blob = new Blob([scadaSymbolContent], { type: this.symbolData.imageResource.descriptor.mediaType }); + const url = URL.createObjectURL(blob); + linkElement.setAttribute('href', url); + linkElement.setAttribute('download', this.symbolData.imageResource.fileName); + const clickEvent = new MouseEvent('click', + { + view: window, + bubbles: true, + cancelable: false + } + ); + linkElement.dispatchEvent(clickEvent); + } catch (e) { + this.store.dispatch(new ActionNotificationShow({ message: e.message, type: 'error' })); + } } createWidget() { diff --git a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html index bbaf7d88ae..11dffb1b25 100644 --- a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html +++ b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ ( uploadImage ? (isScada ? 'scada.upload-symbol' : 'image.upload-image') : (isScada ? 'scada.update-symbol' : 'image.update-image') ) | translate }}

diff --git a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.ts b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.ts index 86bf9c4cd8..779cc021d4 100644 --- a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.ts +++ b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.ts @@ -41,6 +41,7 @@ import { updateScadaSymbolMetadataInContent } from '@home/components/widget/lib/scada/scada-symbol.models'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; export interface UploadImageDialogData { imageSubType: ResourceSubType; @@ -135,38 +136,48 @@ export class UploadImageDialogComponent extends upload(): void { this.submitted = true; let file: File = this.uploadImageFormGroup.get('file').value; - if (this.uploadImage) { - const title: string = this.uploadImageFormGroup.get('title').value; - if (this.isScada) { - if (!this.scadaSymbolMetadata) { - this.scadaSymbolMetadata = emptyMetadata(); - } - if (this.scadaSymbolMetadata.title !== title) { - this.scadaSymbolMetadata.title = title; + try { + if (this.uploadImage) { + const title: string = this.uploadImageFormGroup.get('title').value; + if (this.isScada) { + if (!this.scadaSymbolMetadata) { + this.scadaSymbolMetadata = emptyMetadata(); + } + if (this.scadaSymbolMetadata.title !== title) { + this.scadaSymbolMetadata.title = title; + } + const newContent = updateScadaSymbolMetadataInContent(this.scadaSymbolContent, this.scadaSymbolMetadata); + file = updateFileContent(file, newContent); } - const newContent = updateScadaSymbolMetadataInContent(this.scadaSymbolContent, this.scadaSymbolMetadata); - file = updateFileContent(file, newContent); - } - forkJoin([ - this.imageService.uploadImage(file, title, this.data.imageSubType), - blobToBase64(file) - ]).subscribe(([imageInfo, base64]) => { - this.dialogRef.close({image: Object.assign(imageInfo, {base64})}); - }); - } else { - if (this.isScada) { - blobToText(file).subscribe(scadaSymbolContent => { - this.dialogRef.close({scadaSymbolContent}); - }); - } else { - const image = this.data.image; forkJoin([ - this.imageService.updateImage(imageResourceType(image), image.resourceKey, file), + this.imageService.uploadImage(file, title, this.data.imageSubType), blobToBase64(file) ]).subscribe(([imageInfo, base64]) => { - this.dialogRef.close({image:Object.assign(imageInfo, {base64})}); + this.dialogRef.close({image: Object.assign(imageInfo, {base64})}); }); + } else { + if (this.isScada) { + blobToText(file).subscribe(scadaSymbolContent => { + this.dialogRef.close({scadaSymbolContent}); + }); + } else { + const image = this.data.image; + forkJoin([ + this.imageService.updateImage(imageResourceType(image), image.resourceKey, file), + blobToBase64(file) + ]).subscribe(([imageInfo, base64]) => { + this.dialogRef.close({image:Object.assign(imageInfo, {base64})}); + }); + } } + } catch (e) { + this.store.dispatch(new ActionNotificationShow({ + message: e.message, + type: 'error', + verticalPosition: 'bottom', + horizontalPosition: 'right', + target: 'uploadRoot' + })); } } } From 84778afc0d8ffe4f4ec9524759600516d97ba1f6 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 11 Apr 2025 10:12:46 +0300 Subject: [PATCH 13/15] UI: Change error position for upload dialog --- .../components/image/upload-image-dialog.component.html | 6 +++--- .../components/image/upload-image-dialog.component.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html index 11dffb1b25..47950e53b7 100644 --- a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html +++ b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

{{ ( uploadImage ? (isScada ? 'scada.upload-symbol' : 'image.upload-image') : (isScada ? 'scada.update-symbol' : 'image.update-image') ) | translate }}

@@ -28,8 +28,8 @@
-
-
+
+
Date: Fri, 11 Apr 2025 12:02:00 +0300 Subject: [PATCH 14/15] Minor refactoring for EdqsStatsService --- .../server/edqs/stats/DefaultEdqsStatsService.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java index 4853920c44..435c7f0884 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java @@ -53,13 +53,12 @@ public class DefaultEdqsStatsService implements EdqsStatsService { private final ConcurrentMap objectCounters = new ConcurrentHashMap<>(); private final ConcurrentMap timers = new ConcurrentHashMap<>(); private final ConcurrentMap counters = new ConcurrentHashMap<>(); - private final ConcurrentMap gauges = new ConcurrentHashMap<>(); @PostConstruct private void init() { - statsFactory.createGauge("edqsGauges", "stringPoolSize", TbStringPool.getPool(), Map::size); - statsFactory.createGauge("edqsGauges", "bytePoolSize", TbBytePool.getPool(), Map::size); - statsFactory.createGauge("edqsGauges", "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size); + statsFactory.createGauge("edqsMapGauges", "stringPoolSize", TbStringPool.getPool(), Map::size); + statsFactory.createGauge("edqsMapGauges", "bytePoolSize", TbBytePool.getPool(), Map::size); + statsFactory.createGauge("edqsMapGauges", "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size); } @Override @@ -124,10 +123,6 @@ public class DefaultEdqsStatsService implements EdqsStatsService { return counters.computeIfAbsent(name, __ -> statsFactory.createStatsCounter("edqsCounters", name)); } - private AtomicInteger getGauge(String name) { - return gauges.computeIfAbsent(name, __ -> statsFactory.createGauge("edqsGauges", name, new AtomicInteger())); - } - private AtomicInteger getObjectGauge(ObjectType objectType) { return objectCounters.computeIfAbsent(objectType, type -> statsFactory.createGauge("edqsGauges", "objectsCount", new AtomicInteger(), "objectType", type.name())); From ad40848a25efa3c3c06b58ef563e4b327a205d66 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 11 Apr 2025 12:29:20 +0300 Subject: [PATCH 15/15] Fix DefaultTbCoreConsumerServiceTest --- .../server/service/queue/DefaultTbCoreConsumerService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index e115e4f7b5..fe2d291765 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -116,6 +117,7 @@ import java.util.function.Function; import java.util.stream.Collectors; @Service +@Slf4j @TbCoreComponent public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService {