Browse Source

Merge pull request #2839 from vzikratyi-tb/fix-yml-config

Refactored k8s configs (made 'basic' and 'high-availability' deployment modes)
pull/2878/head
VoBa 6 years ago
committed by GitHub
parent
commit
edb9ef7a3f
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 5
      k8s/.env
  2. 31
      k8s/README.md
  3. 26
      k8s/basic/tb-node-cache-configmap.yml
  4. 181
      k8s/basic/thirdparty.yml
  5. 0
      k8s/common/cassandra.yml
  6. 0
      k8s/common/database-setup.yml
  7. 2
      k8s/common/postgres.yml
  8. 0
      k8s/common/tb-coap-transport-configmap.yml
  9. 0
      k8s/common/tb-http-transport-configmap.yml
  10. 0
      k8s/common/tb-mqtt-transport-configmap.yml
  11. 0
      k8s/common/tb-namespace.yml
  12. 0
      k8s/common/tb-node-cassandra-configmap.yml
  13. 0
      k8s/common/tb-node-configmap.yml
  14. 0
      k8s/common/tb-node-postgres-configmap.yml
  15. 98
      k8s/common/tb-node.yml
  16. 103
      k8s/common/thingsboard.yml
  17. 27
      k8s/high-availability/tb-node-cache-configmap.yml
  18. 2
      k8s/high-availability/thirdparty.yml
  19. 7
      k8s/k8s-delete-resources.sh
  20. 4
      k8s/k8s-delete-thirdparty.sh
  21. 16
      k8s/k8s-deploy-resources.sh
  22. 27
      k8s/k8s-deploy-thirdparty.sh
  23. 25
      k8s/k8s-install-tb.sh
  24. 2
      k8s/k8s-upgrade-tb.sh

5
k8s/.env

@ -1,5 +1,8 @@
# Can be either basic (with single instance of Zookeeper, Kafka and Redis) or high-availability (with Zookeeper, Kafka and Redis in cluster modes).
# According to the deployment type corresponding kubernetes resources will be deployed (see content of the directories ./basic and ./high-availability for details).
DEPLOYMENT_TYPE=high-availability
# Database used by ThingsBoard, can be either postgres (PostgreSQL) or cassandra (Cassandra).
# According to the database type corresponding kubernetes resources will be deployed (see postgres.yml, cassandra.yml for details).
DATABASE=cassandra
DATABASE=postgres

31
k8s/README.md

@ -20,7 +20,7 @@ $ minikube addons enable ingress
## Installation
Before performing initial installation you can configure the type of database to be used with ThingsBoard.
Before performing initial installation you can configure the type of database to be used with ThingsBoard and the type of deployment.
In order to set database type change the value of `DATABASE` variable in `.env` file to one of the following:
- `postgres` - use PostgreSQL database;
@ -28,6 +28,13 @@ In order to set database type change the value of `DATABASE` variable in `.env`
**NOTE**: According to the database type corresponding kubernetes resources will be deployed (see `postgres.yml`, `cassandra.yml` for details).
In order to set deployment type change the value of `DEPLOYMENT_TYPE` variable in `.env` file to one of the following:
- `basic` - start up with single instance of Zookeeper, Kafka and Redis;
- `high-availability` - start up with Zookeeper, Kafka and Redis in cluster modes;
**NOTE**: According to the deployment type corresponding kubernetes resources will be deployed (see content of the directories `./basic` and `./high-availability` for details).
Execute the following command to run installation:
`
@ -46,19 +53,7 @@ Execute the following command to deploy thirdparty resources:
$ ./k8s-deploy-thirdparty.sh
`
Get list of the running tb-redis pods and verify that all of them are in running state:
`
$ kubectl get pods -l app=tb-redis
`
Execute the following command to create redis cluster:
`
$ kubectl exec -it tb-redis-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -l app=tb-redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ')
`
Type **'yes'** when prompted.
Type **'yes'** when prompted, if you are running ThingsBoard in `high-availability` `DEPLOYMENT_TYPE` for the first time and don't have configured Redis cluster.
Execute the following command to deploy resources:
@ -102,12 +97,18 @@ Or use `kubectl get services` to see the state of all the services.
Or use `kubectl get deployments` to see the state of all the deployments.
See [kubectl Cheat Sheet](https://kubernetes.io/docs/reference/kubectl/cheatsheet/) command reference for details.
Execute the following command to delete all deployed microservices:
Execute the following command to delete all ThingsBoard microservices:
`
$ ./k8s-delete-resources.sh
`
Execute the following command to delete all thirdparty microservices:
`
$ ./k8s-delete-thirdparty.sh
`
Execute the following command to delete all resources (including database):
`

26
k8s/basic/tb-node-cache-configmap.yml

@ -0,0 +1,26 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
apiVersion: v1
kind: ConfigMap
metadata:
name: tb-node-cache-config
namespace: thingsboard
labels:
name: tb-node-cache-config
data:
CACHE_TYPE: redis
REDIS_HOST: tb-redis

181
k8s/basic/thirdparty.yml

@ -0,0 +1,181 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
apiVersion: apps/v1
kind: Deployment
metadata:
name: zookeeper
namespace: thingsboard
spec:
selector:
matchLabels:
app: zookeeper
template:
metadata:
labels:
app: zookeeper
spec:
containers:
- name: server
imagePullPolicy: Always
image: zookeeper:3.5
ports:
- containerPort: 2181
readinessProbe:
periodSeconds: 5
tcpSocket:
port: 2181
livenessProbe:
initialDelaySeconds: 15
periodSeconds: 5
tcpSocket:
port: 2181
env:
- name: ZOO_MY_ID
value: "1"
- name: ZOO_SERVERS
value: "server.1=0.0.0.0:2888:3888;0.0.0.0:2181"
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: zookeeper
namespace: thingsboard
spec:
type: ClusterIP
selector:
app: zookeeper
ports:
- name: zk-port
port: 2181
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tb-kafka
namespace: thingsboard
spec:
selector:
matchLabels:
app: tb-kafka
template:
metadata:
labels:
app: tb-kafka
spec:
containers:
- name: server
imagePullPolicy: Always
image: wurstmeister/kafka:2.12-2.2.1
ports:
- containerPort: 9092
readinessProbe:
periodSeconds: 20
tcpSocket:
port: 9092
livenessProbe:
initialDelaySeconds: 25
periodSeconds: 5
tcpSocket:
port: 9092
env:
- name: KAFKA_ZOOKEEPER_CONNECT
value: "zookeeper:2181"
- name: KAFKA_LISTENERS
value: "INSIDE://:9093,OUTSIDE://:9092"
- name: KAFKA_ADVERTISED_LISTENERS
value: "INSIDE://:9093,OUTSIDE://tb-kafka:9092"
- name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
value: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT"
- name: KAFKA_INTER_BROKER_LISTENER_NAME
value: "INSIDE"
- name: KAFKA_CREATE_TOPICS
value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_rule_engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
- name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
value: "false"
- name: KAFKA_LOG_RETENTION_BYTES
value: "1073741824"
- name: KAFKA_LOG_SEGMENT_BYTES
value: "268435456"
- name: KAFKA_LOG_RETENTION_MS
value: "300000"
- name: KAFKA_LOG_CLEANUP_POLICY
value: "delete"
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: tb-kafka
namespace: thingsboard
spec:
type: ClusterIP
selector:
app: tb-kafka
ports:
- name: tb-kafka-port
port: 9092
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tb-redis
namespace: thingsboard
spec:
selector:
matchLabels:
app: tb-redis
template:
metadata:
labels:
app: tb-redis
spec:
containers:
- name: server
imagePullPolicy: Always
image: redis:4.0
ports:
- containerPort: 6379
readinessProbe:
periodSeconds: 5
tcpSocket:
port: 6379
livenessProbe:
periodSeconds: 5
tcpSocket:
port: 6379
volumeMounts:
- mountPath: /data
name: redis-data
volumes:
- name: redis-data
emptyDir: {}
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: tb-redis
namespace: thingsboard
spec:
type: ClusterIP
selector:
app: tb-redis
ports:
- name: tb-redis-port
port: 6379
---

0
k8s/cassandra.yml → k8s/common/cassandra.yml

0
k8s/database-setup.yml → k8s/common/database-setup.yml

2
k8s/postgres.yml → k8s/common/postgres.yml

@ -51,7 +51,7 @@ spec:
containers:
- name: postgres
imagePullPolicy: Always
image: postgres:10
image: postgres:12
ports:
- containerPort: 5432
name: postgres

0
k8s/tb-coap-transport-configmap.yml → k8s/common/tb-coap-transport-configmap.yml

0
k8s/tb-http-transport-configmap.yml → k8s/common/tb-http-transport-configmap.yml

0
k8s/tb-mqtt-transport-configmap.yml → k8s/common/tb-mqtt-transport-configmap.yml

0
k8s/tb-namespace.yml → k8s/common/tb-namespace.yml

0
k8s/tb-node-cassandra-configmap.yml → k8s/common/tb-node-cassandra-configmap.yml

0
k8s/tb-node-configmap.yml → k8s/common/tb-node-configmap.yml

0
k8s/tb-node-postgres-configmap.yml → k8s/common/tb-node-postgres-configmap.yml

98
k8s/common/tb-node.yml

@ -0,0 +1,98 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
apiVersion: apps/v1
kind: Deployment
metadata:
name: tb-node
namespace: thingsboard
spec:
replicas: 2
selector:
matchLabels:
app: tb-node
template:
metadata:
labels:
app: tb-node
spec:
volumes:
- name: tb-node-config
configMap:
name: tb-node-config
items:
- key: conf
path: thingsboard.conf
- key: logback
path: logback.xml
containers:
- name: server
imagePullPolicy: Always
image: thingsboard/tb-node:latest
ports:
- containerPort: 8080
name: http
- containerPort: 9001
name: rpc
env:
- name: TB_SERVICE_ID
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: TB_SERVICE_TYPE
value: "monolith"
- name: TB_QUEUE_TYPE
value: "kafka"
- name: ZOOKEEPER_ENABLED
value: "true"
- name: ZOOKEEPER_URL
value: "zookeeper:2181"
- name: TB_KAFKA_SERVERS
value: "tb-kafka:9092"
- name: JS_EVALUATOR
value: "remote"
- name: TRANSPORT_TYPE
value: "remote"
- name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE
value: "false"
envFrom:
- configMapRef:
name: tb-node-db-config
- configMapRef:
name: tb-node-cache-config
volumeMounts:
- mountPath: /config
name: tb-node-config
livenessProbe:
httpGet:
path: /login
port: http
initialDelaySeconds: 300
timeoutSeconds: 10
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: tb-node
namespace: thingsboard
spec:
type: ClusterIP
selector:
app: tb-node
ports:
- port: 8080
name: http

103
k8s/thingsboard.yml → k8s/common/thingsboard.yml

@ -35,7 +35,7 @@ spec:
image: thingsboard/tb-js-executor:latest
env:
- name: REMOTE_JS_EVAL_REQUEST_TOPIC
value: "js.eval.requests"
value: "js_eval.requests"
- name: TB_KAFKA_SERVERS
value: "tb-kafka:9092"
- name: LOGGER_LEVEL
@ -52,93 +52,6 @@ spec:
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tb-node
namespace: thingsboard
spec:
replicas: 2
selector:
matchLabels:
app: tb-node
template:
metadata:
labels:
app: tb-node
spec:
volumes:
- name: tb-node-config
configMap:
name: tb-node-config
items:
- key: conf
path: thingsboard.conf
- key: logback
path: logback.xml
containers:
- name: server
imagePullPolicy: Always
image: thingsboard/tb-node:latest
ports:
- containerPort: 8080
name: http
- containerPort: 9001
name: rpc
env:
- name: TB_SERVICE_ID
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: TB_SERVICE_TYPE
value: "monolith"
- name: ZOOKEEPER_ENABLED
value: "true"
- name: ZOOKEEPER_URL
value: "zookeeper:2181"
- name: TB_KAFKA_SERVERS
value: "tb-kafka:9092"
- name: JS_EVALUATOR
value: "remote"
- name: TRANSPORT_TYPE
value: "remote"
- name: CACHE_TYPE
value: "redis"
- name: REDIS_HOST
value: "tb-redis"
- name: REDIS_CONNECTION_TYPE
value: "cluster"
- name: REDIS_NODES
value: "tb-redis:6379"
- name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE
value: "false"
envFrom:
- configMapRef:
name: tb-node-db-config
volumeMounts:
- mountPath: /config
name: tb-node-config
livenessProbe:
httpGet:
path: /login
port: http
initialDelaySeconds: 120
timeoutSeconds: 10
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: tb-node
namespace: thingsboard
spec:
type: ClusterIP
selector:
app: tb-node
ports:
- port: 8080
name: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tb-mqtt-transport
namespace: thingsboard
@ -173,8 +86,8 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: TB_SERVICE_TYPE
value: "monolith"
- name: TB_QUEUE_TYPE
value: "kafka"
- name: MQTT_BIND_ADDRESS
value: "0.0.0.0"
- name: MQTT_BIND_PORT
@ -191,6 +104,7 @@ spec:
tcpSocket:
port: 1883
livenessProbe:
initialDelaySeconds: 120
periodSeconds: 20
tcpSocket:
port: 1883
@ -246,8 +160,8 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: TB_SERVICE_TYPE
value: "monolith"
- name: TB_QUEUE_TYPE
value: "kafka"
- name: HTTP_BIND_ADDRESS
value: "0.0.0.0"
- name: HTTP_BIND_PORT
@ -264,6 +178,7 @@ spec:
tcpSocket:
port: 8080
livenessProbe:
initialDelaySeconds: 120
periodSeconds: 20
tcpSocket:
port: 8080
@ -319,8 +234,8 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: TB_SERVICE_TYPE
value: "monolith"
- name: TB_QUEUE_TYPE
value: "kafka"
- name: COAP_BIND_ADDRESS
value: "0.0.0.0"
- name: COAP_BIND_PORT

27
k8s/high-availability/tb-node-cache-configmap.yml

@ -0,0 +1,27 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
apiVersion: v1
kind: ConfigMap
metadata:
name: tb-node-cache-config
namespace: thingsboard
labels:
name: tb-node-cache-config
data:
CACHE_TYPE: redis
REDIS_CONNECTION_TYPE: cluster
REDIS_NODES: tb-redis:6379

2
k8s/thirdparty.yml → k8s/high-availability/thirdparty.yml

@ -169,7 +169,7 @@ spec:
- name: KAFKA_CONTROLLER_SHUTDOWN_ENABLE
value: "true"
- name: KAFKA_CREATE_TOPICS
value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_rule_engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
- name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
value: "false"
- name: KAFKA_LOG_RETENTION_BYTES

7
k8s/k8s-delete-resources.sh

@ -17,6 +17,9 @@
set -e
source .env
kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
kubectl delete -f thingsboard.yml
kubectl delete -f thirdparty.yml
kubectl delete -f common/thingsboard.yml
kubectl delete -f common/tb-node.yml

4
k8s/k8s-delete-thirdparty.sh

@ -17,5 +17,7 @@
set -e
source .env
kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
kubectl delete -f thirdparty.yml
kubectl delete -f $DEPLOYMENT_TYPE/thirdparty.yml

16
k8s/k8s-deploy-resources.sh

@ -17,10 +17,14 @@
set -e
kubectl apply -f tb-namespace.yml
source .env
kubectl apply -f common/tb-namespace.yml
kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
kubectl apply -f tb-node-configmap.yml
kubectl apply -f tb-mqtt-transport-configmap.yml
kubectl apply -f tb-http-transport-configmap.yml
kubectl apply -f tb-coap-transport-configmap.yml
kubectl apply -f thingsboard.yml
kubectl apply -f common/tb-node-configmap.yml
kubectl apply -f common/tb-mqtt-transport-configmap.yml
kubectl apply -f common/tb-http-transport-configmap.yml
kubectl apply -f common/tb-coap-transport-configmap.yml
kubectl apply -f common/thingsboard.yml
kubectl apply -f $DEPLOYMENT_TYPE/tb-node-cache-configmap.yml
kubectl apply -f common/tb-node.yml

27
k8s/k8s-deploy-thirdparty.sh

@ -17,6 +17,29 @@
set -e
kubectl apply -f tb-namespace.yml
source .env
kubectl apply -f common/tb-namespace.yml
kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
kubectl apply -f thirdparty.yml
kubectl apply -f $DEPLOYMENT_TYPE/thirdparty.yml
if [ "$DEPLOYMENT_TYPE" == "high-availability" ]; then
echo -n "waiting for all redis pods to be ready";
while [[ $(kubectl get pods tb-redis-5 -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}' 2>/dev/null) != "True" ]];
do
echo -n "." && sleep 5;
done
if [[ $(kubectl exec -it tb-redis-0 -- redis-cli cluster info 2>&1 | head -n 1) =~ "cluster_state:ok" ]]
then
echo "redis cluster is already configured"
else
echo "starting redis cluster"
redisNodes=$(kubectl get pods -l app=tb-redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ')
kubectl exec -it tb-redis-0 -- redis-cli --cluster create --cluster-replicas 1 $redisNodes
fi
fi

25
k8s/k8s-install-tb.sh

@ -19,8 +19,8 @@ function installTb() {
loadDemo=$1
kubectl apply -f tb-node-configmap.yml
kubectl apply -f database-setup.yml &&
kubectl apply -f common/tb-node-configmap.yml
kubectl apply -f common/database-setup.yml &&
kubectl wait --for=condition=Ready pod/tb-db-setup --timeout=120s &&
kubectl exec tb-db-setup -- sh -c 'export INSTALL_TB=true; export LOAD_DEMO='"$loadDemo"'; start-tb-node.sh; touch /tmp/install-finished;'
@ -30,16 +30,16 @@ function installTb() {
function installPostgres() {
kubectl apply -f postgres.yml
kubectl apply -f tb-node-postgres-configmap.yml
kubectl apply -f common/postgres.yml
kubectl apply -f common/tb-node-postgres-configmap.yml
kubectl rollout status deployment/postgres
}
function installCassandra() {
kubectl apply -f cassandra.yml
kubectl apply -f tb-node-cassandra-configmap.yml
kubectl apply -f common/cassandra.yml
kubectl apply -f common/tb-node-cassandra-configmap.yml
kubectl rollout status statefulset/cassandra
@ -75,9 +75,19 @@ fi
source .env
kubectl apply -f tb-namespace.yml
kubectl apply -f common/tb-namespace.yml
kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
case $DEPLOYMENT_TYPE in
basic)
;;
high-availability)
;;
*)
echo "Unknown DEPLOYMENT_TYPE value specified: '${DEPLOYMENT_TYPE}'. Should be either basic or high-availability." >&2
exit 1
esac
case $DATABASE in
postgres)
installPostgres
@ -91,3 +101,4 @@ case $DATABASE in
echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or cassandra." >&2
exit 1
esac

2
k8s/k8s-upgrade-tb.sh

@ -36,7 +36,7 @@ else
fromVersion="${FROM_VERSION// }"
fi
kubectl apply -f database-setup.yml &&
kubectl apply -f common/database-setup.yml &&
kubectl wait --for=condition=Ready pod/tb-db-setup --timeout=120s &&
kubectl exec tb-db-setup -- sh -c 'export UPGRADE_TB=true; export FROM_VERSION='"$fromVersion"'; start-tb-node.sh; touch /tmp/install-finished;'

Loading…
Cancel
Save