Merge pull request #6021 from Budibase/hotfixes/perf-bg

Hotfixes/perf bg
4 years ago · 5ac4bc8782
10 changed files with 196 additions and 53 deletions
--- a/packages/backend-core/src/utils.js
+++ b/packages/backend-core/src/utils.js
@ -197,11 +197,16 @@ exports.getBuildersCount = async () => {
  return builders.length
 }

-exports.saveUser = async (
+const DEFAULT_SAVE_USER = {
+  hashPassword: true,
+  requirePassword: true,
+  bulkCreate: false,
+}
+
+exports.internalSaveUser = async (
  user,
  tenantId,
-  hashPassword = true,
-  requirePassword = true
+  { hashPassword, requirePassword, bulkCreate } = DEFAULT_SAVE_USER
 ) => {
  if (!tenantId) {
    throw "No tenancy specified."
@ -213,7 +218,10 @@ exports.saveUser = async (
    let { email, password, _id } = user
    // make sure another user isn't using the same email
    let dbUser
-    if (email) {
+    // user can't exist in bulk creation
+    if (bulkCreate) {
+      dbUser = null
+    } else if (email) {
      // check budibase users inside the tenant
      dbUser = await exports.getGlobalUserByEmail(email)
      if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) {
@ -267,11 +275,17 @@ exports.saveUser = async (
      user.status = UserStatus.ACTIVE
    }
    try {
-      const response = await db.put({
+      const putOpts = {
        password: hashedPassword,
        ...user,
-      })
-      await tryAddTenant(tenantId, _id, email)
+      }
+      if (bulkCreate) {
+        return putOpts
+      }
+      const response = await db.put(putOpts)
+      if (env.MULTI_TENANCY) {
+        await tryAddTenant(tenantId, _id, email)
+      }
      await userCache.invalidateUser(response.id)
      return {
        _id: response.id,
@ -288,6 +302,19 @@ exports.saveUser = async (
  })
 }

+// maintained for api compat, don't want to change function signature
+exports.saveUser = async (
+  user,
+  tenantId,
+  hashPassword = true,
+  requirePassword = true
+) => {
+  return exports.internalSaveUser(user, tenantId, {
+    hashPassword,
+    requirePassword,
+  })
+}
+
 /**
 * Logs a user out from budibase. Re-used across account portal and builder.
 */
--- a/packages/server/scripts/load/users.js
+++ b/packages/server/scripts/load/users.js
@ -0,0 +1,97 @@
+// get the JWT secret etc
+require("../../src/environment")
+require("@budibase/backend-core").init()
+const {
+  getProdAppID,
+  generateGlobalUserID,
+} = require("@budibase/backend-core/db")
+const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { internalSaveUser } = require("@budibase/backend-core/utils")
+const { publicApiUserFix } = require("../../src/utilities/users")
+const { hash } = require("@budibase/backend-core/utils")
+
+const USER_LOAD_NUMBER = 10000
+const BATCH_SIZE = 200
+const PASSWORD = "test"
+const TENANT_ID = "default"
+
+const APP_ID = process.argv[2]
+
+const words = [
+  "test",
+  "testing",
+  "budi",
+  "mail",
+  "age",
+  "risk",
+  "load",
+  "uno",
+  "arm",
+  "leg",
+  "pen",
+  "glass",
+  "box",
+  "chicken",
+  "bottle",
+]
+
+if (!APP_ID) {
+  console.error("Must supply app ID as first CLI option!")
+  process.exit(-1)
+}
+
+const WORD_1 = words[Math.floor(Math.random() * words.length)]
+const WORD_2 = words[Math.floor(Math.random() * words.length)]
+let HASHED_PASSWORD
+
+function generateUser(count) {
+  return {
+    _id: generateGlobalUserID(),
+    password: HASHED_PASSWORD,
+    email: `${WORD_1}${count}@${WORD_2}.com`,
+    roles: {
+      [getProdAppID(APP_ID)]: "BASIC",
+    },
+    status: "active",
+    forceResetPassword: false,
+    firstName: "John",
+    lastName: "Smith",
+  }
+}
+
+async function run() {
+  HASHED_PASSWORD = await hash(PASSWORD)
+  return doInTenant(TENANT_ID, async () => {
+    const db = getGlobalDB()
+    for (let i = 0; i < USER_LOAD_NUMBER; i += BATCH_SIZE) {
+      let userSavePromises = []
+      for (let j = 0; j < BATCH_SIZE; j++) {
+        // like the public API
+        const ctx = publicApiUserFix({
+          request: {
+            body: generateUser(i + j),
+          },
+        })
+        userSavePromises.push(
+          internalSaveUser(ctx.request.body, TENANT_ID, {
+            hashPassword: false,
+            requirePassword: true,
+            bulkCreate: true,
+          })
+        )
+      }
+      const users = await Promise.all(userSavePromises)
+      await db.bulkDocs(users)
+      console.log(`${i + BATCH_SIZE} users have been created.`)
+    }
+  })
+}
+
+run()
+  .then(() => {
+    console.log(`Generated ${USER_LOAD_NUMBER} users!`)
+  })
+  .catch(err => {
+    console.error("Failed for reason: ", err)
+    process.exit(-1)
+  })
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@ -439,6 +439,15 @@ export const destroy = async (ctx: any) => {
 }

 export const sync = async (ctx: any, next: any) => {
+  if (env.DISABLE_AUTO_PROD_APP_SYNC) {
+    ctx.status = 200
+    ctx.body = {
+      message:
+        "App sync disabled. You can reenable with the DISABLE_AUTO_PROD_APP_SYNC environment variable.",
+    }
+    return next()
+  }
+
  const appId = ctx.params.appId
  if (!isDevAppID(appId)) {
    ctx.throw(400, "This action cannot be performed for production apps")
--- a/packages/server/src/api/controllers/public/users.ts
+++ b/packages/server/src/api/controllers/public/users.ts
@ -4,30 +4,9 @@ import {
  readGlobalUser,
  saveGlobalUser,
 } from "../../../utilities/workerRequests"
+import { publicApiUserFix } from "../../../utilities/users"
 import { search as stringSearch } from "./utils"

-const { getProdAppID } = require("@budibase/backend-core/db")
-
-function fixUser(ctx: any) {
-  if (!ctx.request.body) {
-    return ctx
-  }
-  if (!ctx.request.body._id && ctx.params.userId) {
-    ctx.request.body._id = ctx.params.userId
-  }
-  if (!ctx.request.body.roles) {
-    ctx.request.body.roles = {}
-  } else {
-    const newRoles: { [key: string]: string } = {}
-    for (let [appId, role] of Object.entries(ctx.request.body.roles)) {
-      // @ts-ignore
-      newRoles[getProdAppID(appId)] = role
-    }
-    ctx.request.body.roles = newRoles
-  }
-  return ctx
-}
-
 function getUser(ctx: any, userId?: string) {
  if (userId) {
    ctx.params = { userId }
@ -45,7 +24,7 @@ export async function search(ctx: any, next: any) {
 }

 export async function create(ctx: any, next: any) {
-  const response = await saveGlobalUser(fixUser(ctx))
+  const response = await saveGlobalUser(publicApiUserFix(ctx))
  ctx.body = await getUser(ctx, response._id)
  await next()
 }
@ -61,7 +40,7 @@ export async function update(ctx: any, next: any) {
    ...ctx.request.body,
    _rev: user._rev,
  }
-  const response = await saveGlobalUser(fixUser(ctx))
+  const response = await saveGlobalUser(publicApiUserFix(ctx))
  ctx.body = await getUser(ctx, response._id)
  await next()
 }
--- a/packages/server/src/environment.js
+++ b/packages/server/src/environment.js
@ -1,3 +1,5 @@
+const { join } = require("path")
+
 function isTest() {
  return (
    process.env.NODE_ENV === "jest" ||
@ -20,7 +22,9 @@ function isCypress() {

 let LOADED = false
 if (!LOADED && isDev() && !isTest()) {
-  require("dotenv").config()
+  require("dotenv").config({
+    path: join(__dirname, "..", ".env"),
+  })
  LOADED = true
 }

@ -57,6 +61,7 @@ module.exports = {
  JEST_WORKER_ID: process.env.JEST_WORKER_ID,
  BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
  DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
+  DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC,
  // minor
  SALT_ROUNDS: process.env.SALT_ROUNDS,
  LOGGER: process.env.LOGGER,
--- a/packages/server/src/utilities/users.js
+++ b/packages/server/src/utilities/users.js
@ -1,6 +1,7 @@
 const { InternalTables } = require("../db/utils")
 const { getGlobalUser } = require("../utilities/global")
 const { getAppDB } = require("@budibase/backend-core/context")
+const { getProdAppID } = require("@budibase/backend-core/db")

 exports.getFullUser = async (ctx, userId) => {
  const global = await getGlobalUser(userId)
@ -22,3 +23,23 @@ exports.getFullUser = async (ctx, userId) => {
    _id: userId,
  }
 }
+
+exports.publicApiUserFix = ctx => {
+  if (!ctx.request.body) {
+    return ctx
+  }
+  if (!ctx.request.body._id && ctx.params.userId) {
+    ctx.request.body._id = ctx.params.userId
+  }
+  if (!ctx.request.body.roles) {
+    ctx.request.body.roles = {}
+  } else {
+    const newRoles = {}
+    for (let [appId, role] of Object.entries(ctx.request.body.roles)) {
+      // @ts-ignore
+      newRoles[getProdAppID(appId)] = role
+    }
+    ctx.request.body.roles = newRoles
+  }
+  return ctx
+}
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@ -1014,10 +1014,10 @@
  resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
  integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==

-"@budibase/backend-core@1.0.164":
-  version "1.0.164"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6"
-  integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg==
+"@budibase/backend-core@1.0.167":
+  version "1.0.167"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879"
+  integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw==
  dependencies:
    "@techpass/passport-openidconnect" "^0.3.0"
    aws-sdk "^2.901.0"
@ -1091,12 +1091,12 @@
    svelte-flatpickr "^3.2.3"
    svelte-portal "^1.0.0"

-"@budibase/pro@1.0.164":
-  version "1.0.164"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a"
-  integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA==
+"@budibase/pro@1.0.167":
+  version "1.0.167"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84"
+  integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA==
  dependencies:
-    "@budibase/backend-core" "1.0.164"
+    "@budibase/backend-core" "1.0.167"
    node-fetch "^2.6.1"

 "@budibase/standard-components@^0.9.139":
--- a/packages/worker/src/api/controllers/global/configs.js
+++ b/packages/worker/src/api/controllers/global/configs.js
@ -271,13 +271,14 @@ exports.configChecklist = async function (ctx) {
    const oidcConfig = await getScopedFullConfig(db, {
      type: Configs.OIDC,
    })
+
    // They have set up an global user
    const users = await db.allDocs(
      getGlobalUserParams(null, {
        include_docs: true,
+        limit: 1,
      })
    )
-    const adminUser = users.rows.some(row => row.doc.admin)

    ctx.body = {
      apps: {
@ -291,7 +292,7 @@ exports.configChecklist = async function (ctx) {
        link: "/builder/portal/manage/email",
      },
      adminUser: {
-        checked: adminUser,
+        checked: users && users.rows.length >= 1,
        label: "Create your first user",
        link: "/builder/portal/manage/users",
      },
--- a/packages/worker/src/environment.js
+++ b/packages/worker/src/environment.js
@ -1,3 +1,5 @@
+const { join } = require("path")
+
 function isDev() {
  return process.env.NODE_ENV !== "production"
 }
@ -12,7 +14,9 @@ function isTest() {

 let LOADED = false
 if (!LOADED && isDev() && !isTest()) {
-  require("dotenv").config()
+  require("dotenv").config({
+    path: join(__dirname, "..", ".env"),
+  })
  LOADED = true
 }

--- a/packages/worker/yarn.lock
+++ b/packages/worker/yarn.lock
@ -293,10 +293,10 @@
  resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
  integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==

-"@budibase/backend-core@1.0.164":
-  version "1.0.164"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6"
-  integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg==
+"@budibase/backend-core@1.0.167":
+  version "1.0.167"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879"
+  integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw==
  dependencies:
    "@techpass/passport-openidconnect" "^0.3.0"
    aws-sdk "^2.901.0"
@ -321,12 +321,12 @@
    uuid "^8.3.2"
    zlib "^1.0.5"

-"@budibase/pro@1.0.164":
-  version "1.0.164"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a"
-  integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA==
+"@budibase/pro@1.0.167":
+  version "1.0.167"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84"
+  integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA==
  dependencies:
-    "@budibase/backend-core" "1.0.164"
+    "@budibase/backend-core" "1.0.167"
    node-fetch "^2.6.1"

 "@cspotcode/source-map-consumer@0.8.0":