Added prometheus-grafana monitoring to Docker scripts

docker/.env

@@ -19,3 +19,6 @@ TB_VERSION=latest
 DATABASE=postgres
 
 LOAD_BALANCER_NAME=haproxy-certbot
+
+# If enabled Prometheus and Grafana containers are deployed along with other containers
+MONITORING_ENABLED=false

docker/README.md

@@ -100,3 +100,13 @@ $ ./docker-start-services.sh
 Where:
 
 - `FROM_VERSION` - from which version upgrade should be started. See [Upgrade Instructions](https://thingsboard.io/docs/user-guide/install/upgrade-instructions) for valid `fromVersion` values.
+
+
+## Monitoring
+
+If you want to enable monitoring with Prometheus and Grafana you need to set <b>MONITORING_ENABLED</b> environment variable to <b>true</b>.
+After this Prometheus and Grafana containers will be deployed. You can reach Prometheus at `http://localhost:9090` and Grafana at `http://localhost:3000` (default login is `admin` and password `foobar`).
+To change Grafana password you need to update `GF_SECURITY_ADMIN_PASSWORD` environment variable at `./monitoring/grafana/config.monitoring` file.
+Dashboards are loaded from `./monitoring/grafana/provisioning/dashboards` directory.
+
+If you want to add new monitoring jobs for Prometheus update `./monitoring/prometheus/prometheus.yml` file.

docker/compose-utils.sh

@@ -61,6 +61,18 @@ function additionalComposeQueueArgs() {
     echo $ADDITIONAL_COMPOSE_QUEUE_ARGS
 }
 
+function additionalComposeMonitoringArgs() {
+    source .env
+
+    if [ "$MONITORING_ENABLED" = true ]
+    then
+      ADDITIONAL_COMPOSE_MONITORING_ARGS="-f docker-compose.prometheus-grafana.yml"
+      echo $ADDITIONAL_COMPOSE_MONITORING_ARGS
+    else
+      echo ""
+    fi
+}
+
 function additionalStartupServices() {
     source .env
     ADDITIONAL_STARTUP_SERVICES=""

docker/docker-compose.prometheus-grafana.yml

@@ -0,0 +1,47 @@
+#
+# Copyright © 2016-2021 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '2.2'
+
+volumes:
+    prometheus_data: {}
+    grafana_data: {}
+
+services:
+
+  prometheus:
+    image: prom/prometheus:v2.1.0
+    volumes:
+      - ./monitoring/prometheus/:/etc/prometheus/
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    ports:
+      - 9090:9090
+    restart: always
+  grafana:
+    image: grafana/grafana
+    user: "472"
+    depends_on:
+      - prometheus
+    ports:
+      - 3000:3000
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./monitoring/grafana/provisioning/:/etc/grafana/provisioning/
+    env_file:
+      - monitoring/grafana/config.monitoring
+    restart: always

docker/docker-remove-services.sh

@@ -23,4 +23,6 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
 
 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
 
-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS down -v
+ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
+
+docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v

docker/docker-start-services.sh

@@ -23,4 +23,6 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
 
 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
 
-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d
+ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
+
+docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d

docker/docker-stop-services.sh

@@ -23,4 +23,6 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
 
 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
 
-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS stop
+ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
+
+docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop

docker/monitoring/grafana/config.monitoring

@@ -0,0 +1,2 @@
+GF_SECURITY_ADMIN_PASSWORD=foobar
+GF_USERS_ALLOW_SIGN_UP=false

docker/monitoring/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,27 @@
+#
+# Copyright © 2016-2021 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: 1
+
+providers:
+- name: 'Prometheus'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  editable: true
+  options:
+    path: /etc/grafana/provisioning/dashboards

docker/monitoring/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,66 @@
+#
+# Copyright © 2016-2021 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+- name: Prometheus
+  # <string, required> datasource type. Required
+  type: prometheus
+  # <string, required> access mode. direct or proxy. Required
+  access: proxy
+  # <int> org id. will default to orgId 1 if not specified
+  orgId: 1
+  # <string> url
+  url: http://prometheus:9090
+  # <string> database password, if used
+  password:
+  # <string> database user, if used
+  user:
+  # <string> database name, if used
+  database:
+  # <bool> enable/disable basic auth
+  basicAuth: false
+  # <string> basic auth username, if used
+  basicAuthUser:
+  # <string> basic auth password, if used
+  basicAuthPassword:
+  # <bool> enable/disable with credentials headers
+  withCredentials:
+  # <bool> mark as default datasource. Max one per org
+  isDefault: true
+  # <map> fields that will be converted to json and stored in json_data
+  jsonData:
+     graphiteVersion: "1.1"
+     tlsAuth: false
+     tlsAuthWithCACert: false
+  # <string> json object of data that will be encrypted.
+  secureJsonData:
+    tlsCACert: "..."
+    tlsClientCert: "..."
+    tlsClientKey: "..."
+  version: 1
+  # <bool> allow users to edit datasources from the UI.
+  editable: true

docker/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,83 @@
+#
+# Copyright © 2016-2021 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# my global config
+global:
+  scrape_interval:     15s # By default, scrape targets every 15 seconds.
+  evaluation_interval: 15s # By default, scrape targets every 15 seconds.
+  # scrape_timeout is set to the global default (10s).
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+      monitor: 'thingsboard'
+
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+
+  - job_name: 'prometheus'
+    scrape_interval: 5s
+    static_configs:
+     - targets: ['localhost:9090']
+
+  - job_name: 'tb-core1'
+    metrics_path: /actuator/prometheus
+    static_configs:
+      - targets: [ 'tb-core1:8080' ]
+
+  - job_name: 'tb-core2'
+    metrics_path: /actuator/prometheus
+    static_configs:
+      - targets: [ 'tb-core2:8080' ]
+
+  - job_name: 'tb-rule-engine1'
+    metrics_path: /actuator/prometheus
+    static_configs:
+      - targets: [ 'tb-rule-engine1:8080' ]
+
+  - job_name: 'tb-rule-engine2'
+    metrics_path: /actuator/prometheus
+    static_configs:
+      - targets: [ 'tb-rule-engine2:8080' ]
+
+  - job_name: 'tb-mqtt-transport1'
+    metrics_path: /actuator/prometheus
+    static_configs:
+      - targets: [ 'tb-mqtt-transport1:8081' ]
+
+  - job_name: 'tb-mqtt-transport2'
+    metrics_path: /actuator/prometheus
+    static_configs:
+      - targets: [ 'tb-mqtt-transport2:8081' ]
+
+  - job_name: 'tb-http-transport1'
+    metrics_path: /actuator/prometheus
+    static_configs:
+      - targets: [ 'tb-http-transport1:8081' ]
+        -
+  - job_name: 'tb-http-transport2'
+    metrics_path: /actuator/prometheus
+    static_configs:
+      - targets: [ 'tb-http-transport2:8081' ]
+
+  - job_name: 'tb-coap-transport'
+    metrics_path: /actuator/prometheus
+    static_configs:
+      - targets: [ 'tb-coap-transport:8081' ]
+

docker/tb-coap-transport.env

@@ -4,3 +4,9 @@ ZOOKEEPER_URL=zookeeper:2181
 COAP_BIND_ADDRESS=0.0.0.0
 COAP_BIND_PORT=5683
 COAP_TIMEOUT=10000
+
+METRICS_ENABLED=true
+METRICS_ENDPOINTS_EXPOSE=prometheus
+WEB_APPLICATION_ENABLE=true
+WEB_APPLICATION_TYPE=servlet
+HTTP_BIND_PORT=8081

docker/tb-http-transport.env

@@ -4,3 +4,6 @@ ZOOKEEPER_URL=zookeeper:2181
 HTTP_BIND_ADDRESS=0.0.0.0
 HTTP_BIND_PORT=8081
 HTTP_REQUEST_TIMEOUT=60000
+
+METRICS_ENABLED=true
+METRICS_ENDPOINTS_EXPOSE=prometheus

docker/tb-mqtt-transport.env

@@ -4,3 +4,9 @@ ZOOKEEPER_URL=zookeeper:2181
 MQTT_BIND_ADDRESS=0.0.0.0
 MQTT_BIND_PORT=1883
 MQTT_TIMEOUT=10000
+
+METRICS_ENABLED=true
+METRICS_ENDPOINTS_EXPOSE=prometheus
+WEB_APPLICATION_ENABLE=true
+WEB_APPLICATION_TYPE=servlet
+HTTP_BIND_PORT=8081

docker/tb-node.env

@@ -10,3 +10,6 @@ REDIS_HOST=redis
 HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false
 
 TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE=64
+
+METRICS_ENABLED=true
+METRICS_ENDPOINTS_EXPOSE=prometheus