Migration locks and add optional preventRetry option

packages/backend-core/package.json

@@ -19,6 +19,7 @@
     "dotenv": "^16.0.1",
     "emitter-listener": "^1.1.2",
     "ioredis": "^4.27.1",
+    "redlock": "^4.0.0",
     "jsonwebtoken": "^8.5.1",
     "koa-passport": "^4.1.4",
     "lodash": "^4.17.21",
@@ -55,6 +56,7 @@
     "@types/tar-fs": "^2.0.1",
     "@types/uuid": "^8.3.4",
     "@types/semver": "^7.0.0",
+    "@types/redlock": "^4.0.0",
     "ioredis-mock": "^5.5.5",
     "jest": "^27.0.3",
     "koa": "2.7.0",

packages/backend-core/redis.js

@@ -1,4 +1,5 @@
 module.exports = {
   Client: require("./src/redis"),
   utils: require("./src/redis/utils"),
+  clients: require("./src/redis/authRedis"),
 }

packages/backend-core/src/migrations/index.js

@@ -90,6 +90,15 @@ exports.runMigration = async (migration, options = {}) => {
         log(
           `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
         )
+
+        if (migration.preventRetry) {
+          // eagerly set the completion date
+          // so that we never run this migration twice even upon failure
+          doc[migrationName] = Date.now()
+          const response = await db.put(doc)
+          doc._rev = response.rev
+        }
+
         // run the migration with tenant context
         if (migrationType === exports.MIGRATION_TYPES.APP) {
           await context.doInAppContext(db.name, async () => {

packages/backend-core/src/redis/authRedis.js

@@ -1,13 +1,23 @@
 const Client = require("./index")
 const utils = require("./utils")
+const { getRedlock } = require("./redlock")
 
 let userClient, sessionClient, appClient, cacheClient
+let migrationsRedlock
+
+// turn retry off so that only one instance can ever hold the lock
+const migrationsRedlockConfig = { retryCount: 0 }
 
 async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
   sessionClient = await new Client(utils.Databases.SESSIONS).init()
   appClient = await new Client(utils.Databases.APP_METADATA).init()
   cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
+  // pass the underlying ioredis client to redlock
+  migrationsRedlock = getRedlock(
+    cacheClient.getClient(),
+    migrationsRedlockConfig
+  )
 }
 
 process.on("exit", async () => {
@@ -42,4 +52,10 @@ module.exports = {
     }
     return cacheClient
   },
+  getMigrationsRedlock: async () => {
+    if (!migrationsRedlock) {
+      await init()
+    }
+    return migrationsRedlock
+  },
 }

packages/backend-core/src/redis/index.js

@@ -139,6 +139,10 @@ class RedisWrapper {
     this._db = db
   }
 
+  getClient() {
+    return CLIENT
+  }
+
   async init() {
     CLOSED = false
     init()

packages/backend-core/src/redis/redlock.ts

@@ -0,0 +1,21 @@
+import Redlock from "redlock"
+
+export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
+  return new Redlock([redisClient], {
+    // the expected clock drift; for more details
+    // see http://redis.io/topics/distlock
+    driftFactor: 0.01, // multiplied by lock ttl to determine drift time
+
+    // the max number of times Redlock will attempt
+    // to lock a resource before erroring
+    retryCount: opts.retryCount,
+
+    // the time in ms between attempts
+    retryDelay: 200, // time in ms
+
+    // the max time in ms randomly added to retries
+    // to improve performance under high contention
+    // see https://www.awsarchitectureblog.com/2015/03/backoff.html
+    retryJitter: 200, // time in ms
+  })
+}

packages/backend-core/yarn.lock

@@ -829,6 +829,11 @@
   dependencies:
     "@babel/types" "^7.3.0"
 
+"@types/bluebird@*":
+  version "3.5.36"
+  resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.36.tgz#00d9301d4dc35c2f6465a8aec634bb533674c652"
+  integrity sha512-HBNx4lhkxN7bx6P0++W8E289foSu8kO8GCk2unhuVggO+cE7rh9DhZUyPhUxNRG9m+5B5BTKxZQ5ZP92x/mx9Q==
+
 "@types/body-parser@*":
   version "1.19.2"
   resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.2.tgz#aea2059e28b7658639081347ac4fab3de166e6f0"
@@ -895,13 +900,6 @@
   resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1"
   integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==
 
-"@types/ioredis@^4.27.1":
-  version "4.28.10"
-  resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
-  integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==
-  dependencies:
-    "@types/node" "*"
-
 "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1":
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz#4ba8ddb720221f432e443bd5f9117fd22cfd4762"
@@ -993,6 +991,21 @@
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
   integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==
 
+"@types/redis@^2.8.0":
+  version "2.8.32"
+  resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11"
+  integrity sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==
+  dependencies:
+    "@types/node" "*"
+
+"@types/redlock@^4.0.0":
+  version "4.0.3"
+  resolved "https://registry.yarnpkg.com/@types/redlock/-/redlock-4.0.3.tgz#aeab5fe5f0d433a125f6dcf9a884372ac0cddd4b"
+  integrity sha512-mcvvrquwREbAqyZALNBIlf49AL9Aa324BG+J/Dv4TAP8g+nxQMBI4/APNqqS99QEY7VTNT9XvsaczCVGK8uNnQ==
+  dependencies:
+    "@types/bluebird" "*"
+    "@types/redis" "^2.8.0"
+
 "@types/semver@^7.0.0":
   version "7.3.9"
   resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc"
@@ -1397,6 +1410,11 @@ bl@^4.0.3:
     inherits "^2.0.4"
     readable-stream "^3.4.0"
 
+bluebird@^3.7.2:
+  version "3.7.2"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
+  integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
+
 boom@2.x.x:
   version "2.10.1"
   resolved "https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f"
@@ -4835,6 +4853,13 @@ redis-parser@^3.0.0:
   dependencies:
     redis-errors "^1.0.0"
 
+redlock@^4.0.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/redlock/-/redlock-4.2.0.tgz#c26590768559afd5fff76aa1133c94b411ff4f5f"
+  integrity sha512-j+oQlG+dOwcetUt2WJWttu4CZVeRzUrcVcISFmEmfyuwCVSJ93rDT7YSgg7H7rnxwoRyk/jU46kycVka5tW7jA==
+  dependencies:
+    bluebird "^3.7.2"
+
 regenerator-runtime@^0.13.4:
   version "0.13.9"
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz#8925742a98ffd90814988d7566ad30ca3b263b52"

packages/server/src/migrations/index.ts

@@ -1,4 +1,4 @@
-import { migrations } from "@budibase/backend-core"
+import { migrations, redis } from "@budibase/backend-core"
 
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
@@ -15,6 +15,7 @@ export interface Migration {
   opts?: object
   fn: Function
   silent?: boolean
+  preventRetry?: boolean
 }
 
 /**
@@ -66,21 +67,63 @@ export const MIGRATIONS: Migration[] = [
     opts: { all: true },
     fn: backfill.app.run,
     silent: !!env.SELF_HOSTED, // reduce noisy logging
+    preventRetry: !!env.SELF_HOSTED, // only ever run once
   },
   {
     type: migrations.MIGRATION_TYPES.GLOBAL,
     name: "event_global_backfill",
     fn: backfill.global.run,
     silent: !!env.SELF_HOSTED, // reduce noisy logging
+    preventRetry: !!env.SELF_HOSTED, // only ever run once
   },
   {
     type: migrations.MIGRATION_TYPES.INSTALLATION,
     name: "event_installation_backfill",
     fn: backfill.installation.run,
     silent: !!env.SELF_HOSTED, // reduce noisy logging
+    preventRetry: !!env.SELF_HOSTED, // only ever run once
   },
 ]
 
 export const migrate = async (options?: MigrationOptions) => {
-  await migrations.runMigrations(MIGRATIONS, options)
+  if (env.SELF_HOSTED) {
+    // self host runs migrations on startup
+    // make sure only a single instance runs them
+    await migrateWithLock(options)
+  } else {
+    await migrations.runMigrations(MIGRATIONS, options)
+  }
+}
+
+const migrateWithLock = async (options?: MigrationOptions) => {
+  // get a new lock client
+
+  const redlock = await redis.clients.getMigrationsRedlock()
+  // lock for 15 minutes
+  const ttl = 1000 * 60 * 15
+
+  let migrationLock
+
+  // acquire lock
+  try {
+    migrationLock = await redlock.lock("migrations", ttl)
+  } catch (e: any) {
+    if (e.name === "LockError") {
+      return
+    } else {
+      throw e
+    }
+  }
+
+  // run migrations
+  try {
+    await migrations.runMigrations(MIGRATIONS, options)
+  } finally {
+    // release lock
+    try {
+      await migrationLock.unlock()
+    } catch (e) {
+      console.error("unable to release migration lock")
+    }
+  }
 }