Migration locks and add optional preventRetry option

4 years ago · 9f2620dd7a
8 changed files with 130 additions and 9 deletions
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -19,6 +19,7 @@
    "dotenv": "^16.0.1",
    "emitter-listener": "^1.1.2",
    "ioredis": "^4.27.1",
+    "redlock": "^4.0.0",
    "jsonwebtoken": "^8.5.1",
    "koa-passport": "^4.1.4",
    "lodash": "^4.17.21",
@ -55,6 +56,7 @@
    "@types/tar-fs": "^2.0.1",
    "@types/uuid": "^8.3.4",
    "@types/semver": "^7.0.0",
+    "@types/redlock": "^4.0.0",
    "ioredis-mock": "^5.5.5",
    "jest": "^27.0.3",
    "koa": "2.7.0",
--- a/packages/backend-core/redis.js
+++ b/packages/backend-core/redis.js
@ -1,4 +1,5 @@
 module.exports = {
  Client: require("./src/redis"),
  utils: require("./src/redis/utils"),
+  clients: require("./src/redis/authRedis"),
 }
--- a/packages/backend-core/src/migrations/index.js
+++ b/packages/backend-core/src/migrations/index.js
@ -90,6 +90,15 @@ exports.runMigration = async (migration, options = {}) => {
        log(
          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
        )
+
+        if (migration.preventRetry) {
+          // eagerly set the completion date
+          // so that we never run this migration twice even upon failure
+          doc[migrationName] = Date.now()
+          const response = await db.put(doc)
+          doc._rev = response.rev
+        }
+
        // run the migration with tenant context
        if (migrationType === exports.MIGRATION_TYPES.APP) {
          await context.doInAppContext(db.name, async () => {
--- a/packages/backend-core/src/redis/authRedis.js
+++ b/packages/backend-core/src/redis/authRedis.js
@ -1,13 +1,23 @@
 const Client = require("./index")
 const utils = require("./utils")
+const { getRedlock } = require("./redlock")

 let userClient, sessionClient, appClient, cacheClient
+let migrationsRedlock
+
+// turn retry off so that only one instance can ever hold the lock
+const migrationsRedlockConfig = { retryCount: 0 }

 async function init() {
  userClient = await new Client(utils.Databases.USER_CACHE).init()
  sessionClient = await new Client(utils.Databases.SESSIONS).init()
  appClient = await new Client(utils.Databases.APP_METADATA).init()
  cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
+  // pass the underlying ioredis client to redlock
+  migrationsRedlock = getRedlock(
+    cacheClient.getClient(),
+    migrationsRedlockConfig
+  )
 }

 process.on("exit", async () => {
@ -42,4 +52,10 @@ module.exports = {
    }
    return cacheClient
  },
+  getMigrationsRedlock: async () => {
+    if (!migrationsRedlock) {
+      await init()
+    }
+    return migrationsRedlock
+  },
 }
--- a/packages/backend-core/src/redis/index.js
+++ b/packages/backend-core/src/redis/index.js
@ -139,6 +139,10 @@ class RedisWrapper {
    this._db = db
  }

+  getClient() {
+    return CLIENT
+  }
+
  async init() {
    CLOSED = false
    init()
--- a/packages/backend-core/src/redis/redlock.ts
+++ b/packages/backend-core/src/redis/redlock.ts
@ -0,0 +1,21 @@
+import Redlock from "redlock"
+
+export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
+  return new Redlock([redisClient], {
+    // the expected clock drift; for more details
+    // see http://redis.io/topics/distlock
+    driftFactor: 0.01, // multiplied by lock ttl to determine drift time
+
+    // the max number of times Redlock will attempt
+    // to lock a resource before erroring
+    retryCount: opts.retryCount,
+
+    // the time in ms between attempts
+    retryDelay: 200, // time in ms
+
+    // the max time in ms randomly added to retries
+    // to improve performance under high contention
+    // see https://www.awsarchitectureblog.com/2015/03/backoff.html
+    retryJitter: 200, // time in ms
+  })
+}
--- a/packages/backend-core/yarn.lock
+++ b/packages/backend-core/yarn.lock
@ -829,6 +829,11 @@
  dependencies:
    "@babel/types" "^7.3.0"

+"@types/bluebird@*":
+  version "3.5.36"
+  resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.36.tgz#00d9301d4dc35c2f6465a8aec634bb533674c652"
+  integrity sha512-HBNx4lhkxN7bx6P0++W8E289foSu8kO8GCk2unhuVggO+cE7rh9DhZUyPhUxNRG9m+5B5BTKxZQ5ZP92x/mx9Q==
+
 "@types/body-parser@*":
  version "1.19.2"
  resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.2.tgz#aea2059e28b7658639081347ac4fab3de166e6f0"
@ -895,13 +900,6 @@
  resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1"
  integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==

-"@types/ioredis@^4.27.1":
-  version "4.28.10"
-  resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
-  integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==
-  dependencies:
-    "@types/node" "*"
-
 "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1":
  version "2.0.3"
  resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz#4ba8ddb720221f432e443bd5f9117fd22cfd4762"
@ -993,6 +991,21 @@
  resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
  integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==

+"@types/redis@^2.8.0":
+  version "2.8.32"
+  resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11"
+  integrity sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==
+  dependencies:
+    "@types/node" "*"
+
+"@types/redlock@^4.0.0":
+  version "4.0.3"
+  resolved "https://registry.yarnpkg.com/@types/redlock/-/redlock-4.0.3.tgz#aeab5fe5f0d433a125f6dcf9a884372ac0cddd4b"
+  integrity sha512-mcvvrquwREbAqyZALNBIlf49AL9Aa324BG+J/Dv4TAP8g+nxQMBI4/APNqqS99QEY7VTNT9XvsaczCVGK8uNnQ==
+  dependencies:
+    "@types/bluebird" "*"
+    "@types/redis" "^2.8.0"
+
 "@types/semver@^7.0.0":
  version "7.3.9"
  resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc"
@ -1397,6 +1410,11 @@ bl@^4.0.3:
    inherits "^2.0.4"
    readable-stream "^3.4.0"

+bluebird@^3.7.2:
+  version "3.7.2"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
+  integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
+
 boom@2.x.x:
  version "2.10.1"
  resolved "https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f"
@ -4835,6 +4853,13 @@ redis-parser@^3.0.0:
  dependencies:
    redis-errors "^1.0.0"

+redlock@^4.0.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/redlock/-/redlock-4.2.0.tgz#c26590768559afd5fff76aa1133c94b411ff4f5f"
+  integrity sha512-j+oQlG+dOwcetUt2WJWttu4CZVeRzUrcVcISFmEmfyuwCVSJ93rDT7YSgg7H7rnxwoRyk/jU46kycVka5tW7jA==
+  dependencies:
+    bluebird "^3.7.2"
+
 regenerator-runtime@^0.13.4:
  version "0.13.9"
  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz#8925742a98ffd90814988d7566ad30ca3b263b52"
--- a/packages/server/src/migrations/index.ts
+++ b/packages/server/src/migrations/index.ts
@ -1,4 +1,4 @@
-import { migrations } from "@budibase/backend-core"
+import { migrations, redis } from "@budibase/backend-core"

 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
@ -15,6 +15,7 @@ export interface Migration {
  opts?: object
  fn: Function
  silent?: boolean
+  preventRetry?: boolean
 }

 /**
@ -66,21 +67,63 @@ export const MIGRATIONS: Migration[] = [
    opts: { all: true },
    fn: backfill.app.run,
    silent: !!env.SELF_HOSTED, // reduce noisy logging
+    preventRetry: !!env.SELF_HOSTED, // only ever run once
  },
  {
    type: migrations.MIGRATION_TYPES.GLOBAL,
    name: "event_global_backfill",
    fn: backfill.global.run,
    silent: !!env.SELF_HOSTED, // reduce noisy logging
+    preventRetry: !!env.SELF_HOSTED, // only ever run once
  },
  {
    type: migrations.MIGRATION_TYPES.INSTALLATION,
    name: "event_installation_backfill",
    fn: backfill.installation.run,
    silent: !!env.SELF_HOSTED, // reduce noisy logging
+    preventRetry: !!env.SELF_HOSTED, // only ever run once
  },
 ]

 export const migrate = async (options?: MigrationOptions) => {
-  await migrations.runMigrations(MIGRATIONS, options)
+  if (env.SELF_HOSTED) {
+    // self host runs migrations on startup
+    // make sure only a single instance runs them
+    await migrateWithLock(options)
+  } else {
+    await migrations.runMigrations(MIGRATIONS, options)
+  }
+}
+
+const migrateWithLock = async (options?: MigrationOptions) => {
+  // get a new lock client
+
+  const redlock = await redis.clients.getMigrationsRedlock()
+  // lock for 15 minutes
+  const ttl = 1000 * 60 * 15
+
+  let migrationLock
+
+  // acquire lock
+  try {
+    migrationLock = await redlock.lock("migrations", ttl)
+  } catch (e: any) {
+    if (e.name === "LockError") {
+      return
+    } else {
+      throw e
+    }
+  }
+
+  // run migrations
+  try {
+    await migrations.runMigrations(MIGRATIONS, options)
+  } finally {
+    // release lock
+    try {
+      await migrationLock.unlock()
+    } catch (e) {
+      console.error("unable to release migration lock")
+    }
+  }
 }