diff --git a/application/src/main/conf/logback.xml b/application/src/main/conf/logback.xml index 6d2c95ee84..284af71953 100644 --- a/application/src/main/conf/logback.xml +++ b/application/src/main/conf/logback.xml @@ -37,7 +37,7 @@ - + diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 65b5f10407..f2bfbc9478 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -192,7 +192,7 @@ public class EntitiesVersionControlController extends BaseController { @GetMapping(value = "/version/{requestId}/status") public VersionCreationResult getVersionCreateRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true) @PathVariable UUID requestId) throws Exception { - accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE); return versionControlService.getVersionCreateStatus(getCurrentUser(), requestId); } @@ -469,7 +469,7 @@ public class EntitiesVersionControlController extends BaseController { @GetMapping(value = "/entity/{requestId}/status") public VersionLoadResult getVersionLoadRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true) @PathVariable UUID requestId) throws Exception { - accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE); return versionControlService.getVersionLoadStatus(getCurrentUser(), requestId); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 6aac90b281..d731ef1536 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -159,6 +159,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer } else { return ((Invocable) engine).invokeFunction(functionName, args); } + } catch (ScriptException e) { + throw new ExecutionException(e); } catch (Exception e) { onScriptExecutionError(scriptId, e, functionName); throw new ExecutionException(e); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 3bee5ca720..86b364afd4 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -57,6 +57,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Value("${queue.js.max_requests_timeout}") private long maxRequestsTimeout; + @Value("${queue.js.max_exec_requests_timeout:2000}") + private long maxExecRequestsTimeout; + @Getter @Value("${js.remote.max_errors}") private int maxErrors; @@ -170,7 +173,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setScriptIdMSB(scriptId.getMostSignificantBits()) .setScriptIdLSB(scriptId.getLeastSignificantBits()) .setFunctionName(functionName) - .setTimeout((int) maxRequestsTimeout) + .setTimeout((int) maxExecRequestsTimeout) .setScriptBody(scriptBody); for (Object arg : args) { @@ -197,7 +200,6 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override public void onFailure(Throwable t) { - onScriptExecutionError(scriptId, t, scriptBody); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { queueTimeoutMsgs.incrementAndGet(); } @@ -212,8 +214,14 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { return invokeResult.getResult(); } else { final RuntimeException e = new RuntimeException(invokeResult.getErrorDetails()); - onScriptExecutionError(scriptId, e, scriptBody); - log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); + if (JsInvokeProtos.JsInvokeErrorCode.TIMEOUT_ERROR.equals(invokeResult.getErrorCode())) { + onScriptExecutionError(scriptId, e, scriptBody); + queueTimeoutMsgs.incrementAndGet(); + } else if (JsInvokeProtos.JsInvokeErrorCode.COMPILATION_ERROR.equals(invokeResult.getErrorCode())) { + onScriptExecutionError(scriptId, e, scriptBody); + } + queueFailedMsgs.incrementAndGet(); + log.debug("[{}] Failed to invoke function due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); throw e; } }, callbackExecutor); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 005e90001b..b0e7b67f25 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -139,18 +139,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont DonAsynchron.withCallback(pendingCommit, commit -> { cachePut(commit.getTxId(), new VersionCreationResult()); try { - List> gitFutures = new ArrayList<>(); + EntitiesExportCtx theCtx; switch (request.getType()) { case SINGLE_ENTITY: { - handleSingleEntityRequest(new SimpleEntitiesExportCtx(user, commit, (SingleEntityVersionCreateRequest) request)); + var ctx = new SimpleEntitiesExportCtx(user, commit, (SingleEntityVersionCreateRequest) request); + handleSingleEntityRequest(ctx); + theCtx = ctx; break; } case COMPLEX: { - handleComplexRequest(new ComplexEntitiesExportCtx(user, commit, (ComplexVersionCreateRequest) request)); + var ctx = new ComplexEntitiesExportCtx(user, commit, (ComplexVersionCreateRequest) request); + handleComplexRequest(ctx); + theCtx = ctx; break; } + default: + throw new RuntimeException("Unsupported request type: " + request.getType()); } - var resultFuture = Futures.transformAsync(Futures.allAsList(gitFutures), f -> gitServiceQueue.push(commit), executor); + var resultFuture = Futures.transformAsync(Futures.allAsList(theCtx.getFutures()), f -> gitServiceQueue.push(commit), executor); DonAsynchron.withCallback(resultFuture, result -> cachePut(commit.getTxId(), result), e -> processCommitError(user, request, commit, e), executor); } catch (Exception e) { processCommitError(user, request, commit, e); diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 4784ad2d43..941bd7d278 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -27,7 +27,7 @@ - + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index a6fe1b97f0..b7a0e9b52c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1051,6 +1051,8 @@ queue: max_eval_requests_timeout: "${REMOTE_JS_MAX_EVAL_REQUEST_TIMEOUT:60000}" # JS max request timeout max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS execution max request timeout + max_exec_requests_timeout: "${REMOTE_JS_MAX_EXEC_REQUEST_TIMEOUT:2000}" # JS response poll interval response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" rule-engine: diff --git a/docker/tb-node/conf/logback.xml b/docker/tb-node/conf/logback.xml index bc694d704d..c17e8a43cf 100644 --- a/docker/tb-node/conf/logback.xml +++ b/docker/tb-node/conf/logback.xml @@ -42,6 +42,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/tb-transports/coap/conf/logback.xml b/docker/tb-transports/coap/conf/logback.xml index 1a3e8ef6b3..65d13a55cf 100644 --- a/docker/tb-transports/coap/conf/logback.xml +++ b/docker/tb-transports/coap/conf/logback.xml @@ -42,6 +42,11 @@ + + + + + diff --git a/docker/tb-transports/http/conf/logback.xml b/docker/tb-transports/http/conf/logback.xml index 05361fe680..a3ae374bb7 100644 --- a/docker/tb-transports/http/conf/logback.xml +++ b/docker/tb-transports/http/conf/logback.xml @@ -42,6 +42,11 @@ + + + + + diff --git a/docker/tb-transports/lwm2m/conf/logback.xml b/docker/tb-transports/lwm2m/conf/logback.xml index d2485675ee..5546167f3d 100644 --- a/docker/tb-transports/lwm2m/conf/logback.xml +++ b/docker/tb-transports/lwm2m/conf/logback.xml @@ -42,6 +42,10 @@ + + + + diff --git a/docker/tb-transports/mqtt/conf/logback.xml b/docker/tb-transports/mqtt/conf/logback.xml index 09bcaea84c..d4e7b811f8 100644 --- a/docker/tb-transports/mqtt/conf/logback.xml +++ b/docker/tb-transports/mqtt/conf/logback.xml @@ -42,6 +42,10 @@ + + + + diff --git a/docker/tb-transports/snmp/conf/logback.xml b/docker/tb-transports/snmp/conf/logback.xml index dc53ddd8a3..0b6fbc726b 100644 --- a/docker/tb-transports/snmp/conf/logback.xml +++ b/docker/tb-transports/snmp/conf/logback.xml @@ -42,6 +42,10 @@ + + + + diff --git a/docker/tb-vc-executor/conf/logback.xml b/docker/tb-vc-executor/conf/logback.xml index dc95b3a885..ebde7cbd69 100644 --- a/docker/tb-vc-executor/conf/logback.xml +++ b/docker/tb-vc-executor/conf/logback.xml @@ -40,8 +40,11 @@ - - + + + + + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 60934bdbbf..ffd2e1f50a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -95,7 +95,7 @@ public class ContainerTestSuite { new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")), - new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.redis-cluster.volumes.yml" : "docker-compose.redis.volumes.yml")) + new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.volumes.yml" : "docker-compose.redis.volumes.yml")) )); Map queueEnv = new HashMap<>(); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index c39932b572..eedf070fac 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -46,7 +46,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; - private final static String JAVA_OPTS = "-Xmx384m -Xss256k"; + private final static String JAVA_OPTS = "-Xmx512m"; private final DockerComposeExecutor dockerCompose; diff --git a/msa/js-executor/api/utils.ts b/msa/js-executor/api/utils.ts index b43a5f05f2..361025f806 100644 --- a/msa/js-executor/api/utils.ts +++ b/msa/js-executor/api/utils.ts @@ -39,12 +39,6 @@ export function isString(value: any): boolean { return typeof value === 'string'; } -export function sleep(ms: number): Promise { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - export function parseJsErrorDetails(err: any): string | undefined { if (!err) { return undefined; diff --git a/msa/js-executor/docker/Dockerfile b/msa/js-executor/docker/Dockerfile index d17138d923..8746e1db54 100644 --- a/msa/js-executor/docker/Dockerfile +++ b/msa/js-executor/docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM node:16.15.1-bullseye-slim +FROM thingsboard/node:16.15.1-bullseye-slim ENV NODE_ENV production ENV DOCKER_MODE true diff --git a/msa/js-executor/queue/awsSqsTemplate.ts b/msa/js-executor/queue/awsSqsTemplate.ts index 36ec9b5270..259d285cf2 100644 --- a/msa/js-executor/queue/awsSqsTemplate.ts +++ b/msa/js-executor/queue/awsSqsTemplate.ts @@ -34,7 +34,6 @@ import { SQSClient } from '@aws-sdk/client-sqs'; import uuid from 'uuid-random'; -import { sleep } from '../api/utils'; export class AwsSqsTemplate implements IQueue { @@ -48,11 +47,11 @@ export class AwsSqsTemplate implements IQueue { private sqsClient: SQSClient; private requestQueueURL: string - private stopped = false; private queueUrls = new Map(); private queueAttributes: { [n: string]: string } = { FifoQueue: 'true' }; + private timer: NodeJS.Timer; name = 'AWS SQS'; @@ -91,40 +90,37 @@ export class AwsSqsTemplate implements IQueue { const params: ReceiveMessageRequest = { MaxNumberOfMessages: 10, QueueUrl: this.requestQueueURL, - WaitTimeSeconds: this.pollInterval / 1000 + WaitTimeSeconds: Math.ceil(this.pollInterval / 10) }; - while (!this.stopped) { - let pollStartTs = new Date().getTime(); - const messagesResponse: ReceiveMessageResult = await this.sqsClient.send(new ReceiveMessageCommand(params)); - const messages = messagesResponse.Messages; - - if (messages && messages.length > 0) { - const entries: DeleteMessageBatchRequestEntry[] = []; - - messages.forEach(message => { - entries.push({ - Id: message.MessageId, - ReceiptHandle: message.ReceiptHandle - }); - messageProcessor.onJsInvokeMessage(JSON.parse(message.Body || '')); + this.timer = setTimeout(() => {this.getAndProcessMessage(messageProcessor, params)}, this.pollInterval); + } + + private async getAndProcessMessage(messageProcessor: JsInvokeMessageProcessor, params: ReceiveMessageRequest) { + const messagesResponse: ReceiveMessageResult = await this.sqsClient.send(new ReceiveMessageCommand(params)); + const messages = messagesResponse.Messages; + + if (messages && messages.length > 0) { + const entries: DeleteMessageBatchRequestEntry[] = []; + + messages.forEach(message => { + entries.push({ + Id: message.MessageId, + ReceiptHandle: message.ReceiptHandle }); + messageProcessor.onJsInvokeMessage(JSON.parse(message.Body || '')); + }); - const deleteBatch: DeleteMessageBatchRequest = { - QueueUrl: this.requestQueueURL, - Entries: entries - }; - try { - await this.sqsClient.send(new DeleteMessageBatchCommand(deleteBatch)) - } catch (err: any) { - this.logger.error("Failed to delete messages from queue.", err.message); - } - } else { - let pollDuration = new Date().getTime() - pollStartTs; - if (pollDuration < this.pollInterval) { - await sleep(this.pollInterval - pollDuration); - } + const deleteBatch: DeleteMessageBatchRequest = { + QueueUrl: this.requestQueueURL, + Entries: entries + }; + try { + await this.sqsClient.send(new DeleteMessageBatchCommand(deleteBatch)) + } catch (err: any) { + this.logger.error("Failed to delete messages from queue.", err.message); } } + this.timer = setTimeout(() => {this.getAndProcessMessage(messageProcessor, params)}, this.pollInterval); } async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise { @@ -182,8 +178,8 @@ export class AwsSqsTemplate implements IQueue { } async destroy(): Promise { - this.stopped = true; this.logger.info('Stopping AWS SQS resources...'); + clearTimeout(this.timer); if (this.sqsClient) { this.logger.info('Stopping AWS SQS client...'); try { diff --git a/msa/tb/docker/logback.xml b/msa/tb/docker/logback.xml index 92fab7bc88..01fc5d30a8 100644 --- a/msa/tb/docker/logback.xml +++ b/msa/tb/docker/logback.xml @@ -42,6 +42,8 @@ + + diff --git a/msa/vc-executor/src/main/conf/logback.xml b/msa/vc-executor/src/main/conf/logback.xml index c0f33852a6..d7468ef257 100644 --- a/msa/vc-executor/src/main/conf/logback.xml +++ b/msa/vc-executor/src/main/conf/logback.xml @@ -35,8 +35,10 @@ + + - + diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml index e25a1995c1..e0a240aee3 100644 --- a/msa/vc-executor/src/main/resources/logback.xml +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -25,11 +25,11 @@ - - + + - + diff --git a/msa/web-ui/docker/Dockerfile b/msa/web-ui/docker/Dockerfile index 66c0ea9893..7b047381ae 100644 --- a/msa/web-ui/docker/Dockerfile +++ b/msa/web-ui/docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM node:16.15.1-bullseye-slim +FROM thingsboard/node:16.15.1-bullseye-slim ENV NODE_ENV production ENV DOCKER_MODE true diff --git a/transport/coap/src/main/conf/logback.xml b/transport/coap/src/main/conf/logback.xml index a769cbe364..d7468ef257 100644 --- a/transport/coap/src/main/conf/logback.xml +++ b/transport/coap/src/main/conf/logback.xml @@ -38,7 +38,7 @@ - + diff --git a/transport/coap/src/main/resources/logback.xml b/transport/coap/src/main/resources/logback.xml index 44ddb3d372..e0a240aee3 100644 --- a/transport/coap/src/main/resources/logback.xml +++ b/transport/coap/src/main/resources/logback.xml @@ -28,9 +28,8 @@ - - + diff --git a/transport/http/src/main/conf/logback.xml b/transport/http/src/main/conf/logback.xml index a769cbe364..d7468ef257 100644 --- a/transport/http/src/main/conf/logback.xml +++ b/transport/http/src/main/conf/logback.xml @@ -38,7 +38,7 @@ - + diff --git a/transport/http/src/main/resources/logback.xml b/transport/http/src/main/resources/logback.xml index 44ddb3d372..80f9525f2e 100644 --- a/transport/http/src/main/resources/logback.xml +++ b/transport/http/src/main/resources/logback.xml @@ -30,7 +30,7 @@ - + diff --git a/transport/lwm2m/src/main/conf/logback.xml b/transport/lwm2m/src/main/conf/logback.xml index a769cbe364..d7468ef257 100644 --- a/transport/lwm2m/src/main/conf/logback.xml +++ b/transport/lwm2m/src/main/conf/logback.xml @@ -38,7 +38,7 @@ - + diff --git a/transport/lwm2m/src/main/resources/logback.xml b/transport/lwm2m/src/main/resources/logback.xml index 44ddb3d372..80f9525f2e 100644 --- a/transport/lwm2m/src/main/resources/logback.xml +++ b/transport/lwm2m/src/main/resources/logback.xml @@ -30,7 +30,7 @@ - + diff --git a/transport/mqtt/src/main/conf/logback.xml b/transport/mqtt/src/main/conf/logback.xml index a769cbe364..d7468ef257 100644 --- a/transport/mqtt/src/main/conf/logback.xml +++ b/transport/mqtt/src/main/conf/logback.xml @@ -38,7 +38,7 @@ - + diff --git a/transport/mqtt/src/main/resources/logback.xml b/transport/mqtt/src/main/resources/logback.xml index 44ddb3d372..80f9525f2e 100644 --- a/transport/mqtt/src/main/resources/logback.xml +++ b/transport/mqtt/src/main/resources/logback.xml @@ -30,7 +30,7 @@ - + diff --git a/transport/snmp/src/main/conf/logback.xml b/transport/snmp/src/main/conf/logback.xml index a769cbe364..d7468ef257 100644 --- a/transport/snmp/src/main/conf/logback.xml +++ b/transport/snmp/src/main/conf/logback.xml @@ -38,7 +38,7 @@ - + diff --git a/transport/snmp/src/main/resources/logback.xml b/transport/snmp/src/main/resources/logback.xml index 44ddb3d372..80f9525f2e 100644 --- a/transport/snmp/src/main/resources/logback.xml +++ b/transport/snmp/src/main/resources/logback.xml @@ -30,7 +30,7 @@ - + diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/layout/dashboard-layout.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/layout/dashboard-layout.component.ts index 374c6d9b3d..fcee0b2f0a 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/layout/dashboard-layout.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/layout/dashboard-layout.component.ts @@ -94,7 +94,9 @@ export class DashboardLayoutComponent extends PageComponent implements ILayoutCo } ngOnInit(): void { - this.rxSubscriptions.push(this.dashboard.dashboardTimewindowChanged.subscribe( + const dashboardTimewindowChanged = this.parentDashboard ? + this.parentDashboard.dashboardTimewindowChanged : this.dashboard.dashboardTimewindowChanged; + this.rxSubscriptions.push(dashboardTimewindowChanged.subscribe( (dashboardTimewindow) => { this.dashboardCtx.dashboardTimewindow = dashboardTimewindow; this.dashboardCtx.runChangeDetection(); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-widget-settings.component.html index 2cf14d066d..3ecc56b602 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-widget-settings.component.html @@ -288,19 +288,19 @@
widgets.chart.custom-legend-settings - + + fxLayoutAlign="center" style="height: 100%;"> {{ 'widgets.chart.enable-custom-legend' | translate }} - + widget-config.advanced-settings -
+
widgets.chart.label-keys-list