Fixes and refactoring (#2761)

* fix sqs js executor and refactored RemoteJsInvokeService

* added REMOTE_JS_MAX_REQUEST_TIMEOUT=20000 for aws-sqs, pubsub, service-bus docker environments

* added REMOTE_JS_MAX_REQUEST_TIMEOUT=20000 for aws-sqs, pubsub, service-bus docker environments

* refactored

* docker-compose.pubsub.yml improvements

* rabbitmq js executor improvements

application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java

@@ -59,22 +59,22 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
     @Value("${js.remote.stats.enabled:false}")
     private boolean statsEnabled;
 
-    private final AtomicInteger kafkaPushedMsgs = new AtomicInteger(0);
-    private final AtomicInteger kafkaInvokeMsgs = new AtomicInteger(0);
-    private final AtomicInteger kafkaEvalMsgs = new AtomicInteger(0);
-    private final AtomicInteger kafkaFailedMsgs = new AtomicInteger(0);
-    private final AtomicInteger kafkaTimeoutMsgs = new AtomicInteger(0);
+    private final AtomicInteger queuePushedMsgs = new AtomicInteger(0);
+    private final AtomicInteger queueInvokeMsgs = new AtomicInteger(0);
+    private final AtomicInteger queueEvalMsgs = new AtomicInteger(0);
+    private final AtomicInteger queueFailedMsgs = new AtomicInteger(0);
+    private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0);
 
     @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}")
     public void printStats() {
         if (statsEnabled) {
-            int pushedMsgs = kafkaPushedMsgs.getAndSet(0);
-            int invokeMsgs = kafkaInvokeMsgs.getAndSet(0);
-            int evalMsgs = kafkaEvalMsgs.getAndSet(0);
-            int failed = kafkaFailedMsgs.getAndSet(0);
-            int timedOut = kafkaTimeoutMsgs.getAndSet(0);
+            int pushedMsgs = queuePushedMsgs.getAndSet(0);
+            int invokeMsgs = queueInvokeMsgs.getAndSet(0);
+            int evalMsgs = queueEvalMsgs.getAndSet(0);
+            int failed = queueFailedMsgs.getAndSet(0);
+            int timedOut = queueTimeoutMsgs.getAndSet(0);
             if (pushedMsgs > 0 || invokeMsgs > 0 || evalMsgs > 0 || failed > 0 || timedOut > 0) {
-                log.info("Kafka JS Invoke Stats: pushed [{}] received [{}] invoke [{}] eval [{}] failed [{}] timedOut [{}]",
+                log.info("Queue JS Invoke Stats: pushed [{}] received [{}] invoke [{}] eval [{}] failed [{}] timedOut [{}]",
                         pushedMsgs, invokeMsgs + evalMsgs, invokeMsgs, evalMsgs, failed, timedOut);
             }
         }
@@ -116,19 +116,19 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
         if (maxRequestsTimeout > 0) {
             future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
         }
-        kafkaPushedMsgs.incrementAndGet();
+        queuePushedMsgs.incrementAndGet();
         Futures.addCallback(future, new FutureCallback<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>>() {
             @Override
             public void onSuccess(@Nullable TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse> result) {
-                kafkaEvalMsgs.incrementAndGet();
+                queueEvalMsgs.incrementAndGet();
             }
 
             @Override
             public void onFailure(Throwable t) {
                 if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) {
-                    kafkaTimeoutMsgs.incrementAndGet();
+                    queueTimeoutMsgs.incrementAndGet();
                 }
-                kafkaFailedMsgs.incrementAndGet();
+                queueFailedMsgs.incrementAndGet();
             }
         }, MoreExecutors.directExecutor());
         return Futures.transform(future, response -> {
@@ -170,20 +170,20 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
         if (maxRequestsTimeout > 0) {
             future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
         }
-        kafkaPushedMsgs.incrementAndGet();
+        queuePushedMsgs.incrementAndGet();
         Futures.addCallback(future, new FutureCallback<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>>() {
             @Override
             public void onSuccess(@Nullable TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse> result) {
-                kafkaInvokeMsgs.incrementAndGet();
+                queueInvokeMsgs.incrementAndGet();
             }
 
             @Override
             public void onFailure(Throwable t) {
                 onScriptExecutionError(scriptId);
                 if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) {
-                    kafkaTimeoutMsgs.incrementAndGet();
+                    queueTimeoutMsgs.incrementAndGet();
                 }
-                kafkaFailedMsgs.incrementAndGet();
+                queueFailedMsgs.incrementAndGet();
             }
         }, MoreExecutors.directExecutor());
         return Futures.transform(future, response -> {

docker/docker-compose.pubsub.yml

@@ -19,10 +19,10 @@ version: '2.2'
 services:
   tb-js-executor:
     env_file:
-      - queue-pubsub.env.env
+      - queue-pubsub.env
   tb-core1:
     env_file:
-      - queue-pubsub.env.env
+      - queue-pubsub.env
     depends_on:
       - zookeeper
       - redis

docker/queue-aws-sqs.env

@@ -2,3 +2,4 @@ TB_QUEUE_TYPE=aws-sqs
 TB_QUEUE_AWS_SQS_ACCESS_KEY_ID=YOUR_KEY
 TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY=YOUR_SECRET
 TB_QUEUE_AWS_SQS_REGION=YOUR_REGION
+REMOTE_JS_MAX_REQUEST_TIMEOUT=60000

docker/queue-pubsub.env

@@ -1,3 +1,4 @@
 TB_QUEUE_TYPE=pubsub
 TB_QUEUE_PUBSUB_PROJECT_ID=YOUR_PROJECT_ID
-TB_QUEUE_PUBSUB_SERVICE_ACCOUNT=YOUR_SERVICE_ACCOUNT
+TB_QUEUE_PUBSUB_SERVICE_ACCOUNT=YOUR_SERVICE_ACCOUNT
+REMOTE_JS_MAX_REQUEST_TIMEOUT=60000

docker/queue-service-bus.env

@@ -1,4 +1,5 @@
 TB_QUEUE_TYPE=service-bus
 TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME=YOUR_NAMESPACE_NAME
 TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME=YOUR_SAS_KEY_NAME
-TB_QUEUE_SERVICE_BUS_SAS_KEY=YOUR_SAS_KEY
+TB_QUEUE_SERVICE_BUS_SAS_KEY=YOUR_SAS_KEY
+REMOTE_JS_MAX_REQUEST_TIMEOUT=60000

msa/js-executor/queue/awsSqsTemplate.js

@@ -100,7 +100,7 @@ function AwsSqsProducer() {
         const params = {
             MaxNumberOfMessages: 10,
             QueueUrl: requestQueueURL,
-            WaitTimeSeconds: poolInterval / 1000
+            WaitTimeSeconds: pollInterval / 1000
         };
         while (!stopped) {
             let pollStartTs = new Date().getTime();

msa/js-executor/queue/rabbitmqTemplate.js

@@ -68,9 +68,8 @@ function RabbitMqProducer() {
 (async () => {
     try {
         logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
-        const url = `amqp://${host}:${port}${vhost}`;
+        const url = `amqp://${username}:${password}@${host}:${port}${vhost}`;
 
-        amqp.credentials.amqplain(username, password);
         connection = await new Promise((resolve, reject) => {
             amqp.connect(url, function (err, connection) {
                 if (err) {