Fixing issues with Redis/Bull and the integration with new redis module.

hosting/docker-compose.yaml

@@ -105,8 +105,6 @@ services:
     restart: always
     image: redis
     command: redis-server --requirepass ${REDIS_PASSWORD}
-    ports:
-      - "${REDIS_PORT}:6379"
     volumes:
       - redis_data:/data
 

hosting/envoy.dev.yaml.hbs

@@ -21,11 +21,6 @@ static_resources:
                   cluster: couchdb-service
                   prefix_rewrite: "/"
 
-              - match: { prefix: "/cache/" }
-                route:
-                  cluster: redis-service
-                  prefix_rewrite: "/"
-
               - match: { prefix: "/api/admin/" }
                 route:
                   cluster: worker-dev
@@ -89,20 +84,6 @@ static_resources:
                 address: couchdb-service
                 port_value: 5984
 
-  - name: redis-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: redis-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: redis-service
-                port_value: 6379
-
   - name: server-dev
     connect_timeout: 0.25s
     type: strict_dns

hosting/envoy.yaml

@@ -41,11 +41,6 @@ static_resources:
                   cluster: worker-service
                   prefix_rewrite: "/"
 
-              - match: { prefix: "/cache/" }
-                route:
-                  cluster: redis-service
-                  prefix_rewrite: "/"
-
               - match: { prefix: "/db/" }
                 route:
                   cluster: couchdb-service
@@ -117,18 +112,3 @@ static_resources:
                 address: couchdb-service
                 port_value: 5984
 
-  - name: redis-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: redis-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: redis-service
-                port_value: 6379
-
-

packages/auth/src/index.js

@@ -28,6 +28,10 @@ module.exports = {
     setDB(pouch)
   },
   db: require("./db/utils"),
+  redis: {
+    client: require("./redis"),
+    utils: require("./redis/utils"),
+  },
   utils: {
     ...require("./utils"),
     ...require("./hashing"),

packages/auth/src/redis/index.js

@@ -1,9 +1,6 @@
 const Redis = require("ioredis")
-const env = require("../environment")
-const { addDbPrefix, removeDbPrefix } = require("./utils")
+const { addDbPrefix, removeDbPrefix, getRedisOptions } = require("./utils")
 
-const CONNECT_TIMEOUT_MS = 10000
-const SLOT_REFRESH_MS = 2000
 const CLUSTERED = false
 
 let CLIENT
@@ -15,21 +12,10 @@ let CLIENT
  */
 function init() {
   return new Promise((resolve, reject) => {
-    const [host, port] = env.REDIS_URL.split(":")
-    const opts = {
-      connectTimeout: CONNECT_TIMEOUT_MS,
-    }
+    const { opts, host, port } = getRedisOptions(CLUSTERED)
     if (CLUSTERED) {
-      opts.redisOptions = {}
-      opts.redisOptions.tls = {}
-      opts.redisOptions.password = env.REDIS_PASSWORD
-      opts.slotsRefreshTimeout = SLOT_REFRESH_MS
-      opts.dnsLookup = (address, callback) => callback(null, address)
-      CLIENT = new Redis.Cluster([{ port, host }])
+      CLIENT = new Redis.Cluster([{ host, port }], opts)
     } else {
-      opts.password = env.REDIS_PASSWORD
-      opts.port = port
-      opts.host = host
       CLIENT = new Redis(opts)
     }
     CLIENT.on("end", err => {

packages/auth/src/redis/utils.js

@@ -1,9 +1,32 @@
+const env = require("../environment")
+
+const SLOT_REFRESH_MS = 2000
+const CONNECT_TIMEOUT_MS = 10000
 const SEPARATOR = "-"
 
 exports.Databases = {
   PW_RESETS: "pwReset",
 }
 
+exports.getRedisOptions = (clustered = false) => {
+  const [host, port] = env.REDIS_URL.split(":")
+  const opts = {
+    connectTimeout: CONNECT_TIMEOUT_MS,
+  }
+  if (clustered) {
+    opts.redisOptions = {}
+    opts.redisOptions.tls = {}
+    opts.redisOptions.password = env.REDIS_PASSWORD
+    opts.slotsRefreshTimeout = SLOT_REFRESH_MS
+    opts.dnsLookup = (address, callback) => callback(null, address)
+  } else {
+    opts.host = host
+    opts.port = port
+    opts.password = env.REDIS_PASSWORD
+  }
+  return { opts, host, port }
+}
+
 exports.addDbPrefix = (db, key) => {
   return `${db}${SEPARATOR}${key}`
 }

packages/server/scripts/dev/manage.js

@@ -37,9 +37,10 @@ async function init() {
     PORT: 4001,
     MINIO_URL: "http://localhost:10000/",
     COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
-    REDIS_URL: "http://localhost:10000/cache/",
+    REDIS_URL: "localhost:6379",
     WORKER_URL: "http://localhost:4002",
     JWT_SECRET: "testsecret",
+    REDIS_PASSWORD: "budibase",
     MINIO_ACCESS_KEY: "budibase",
     MINIO_SECRET_KEY: "budibase",
     COUCH_DB_PASSWORD: "budibase",

packages/server/src/automations/triggers.js

@@ -4,8 +4,10 @@ const Queue = require("bull")
 const { setQueues, BullAdapter } = require("bull-board")
 const { getAutomationParams } = require("../db/utils")
 const { coerce } = require("../utilities/rowProcessor")
+const { utils } = require("@budibase/auth").redis
 
-let automationQueue = new Queue("automationQueue")
+const { opts } = utils.getRedisOptions()
+let automationQueue = new Queue("automationQueue", { redis: opts })
 
 // Set up queues for bull board admin
 setQueues([new BullAdapter(automationQueue)])

packages/worker/scripts/dev/manage.js

@@ -12,6 +12,8 @@ async function init() {
     MINIO_SECRET_KEY: "budibase",
     COUCH_DB_USER: "budibase",
     COUCH_DB_PASSWORD: "budibase",
+    REDIS_URL: "localhost:6379",
+    REDIS_PASSWORD: "budibase",
     MINIO_URL: "http://localhost:10000/",
     COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
   }