merge

packages/auth/README.md

@@ -1 +1,12 @@
-# Budibase Authentication Library
+# Budibase Core backend library
+
+This library contains core functionality, like auth and security features
+which are shared between backend services.
+
+#### Note about top level JS files
+For the purposes of being able to do say `require("@budibase/auth/permissions")` we need to
+specify the exports at the top-level of the module.
+
+For these files they should be limited to a single `require` of the file that should
+be exported and then a single `module.exports = ...` to export the file in
+commonJS.

packages/auth/db.js

@@ -0,0 +1 @@
+module.exports = require("./src/db/utils")

packages/auth/package.json

@@ -11,6 +11,7 @@
     "ioredis": "^4.27.1",
     "jsonwebtoken": "^8.5.1",
     "koa-passport": "^4.1.4",
+    "lodash": "^4.17.21",
     "node-fetch": "^2.6.1",
     "passport-google-auth": "^1.0.2",
     "passport-google-oauth": "^2.0.0",

packages/auth/permissions.js

@@ -0,0 +1 @@
+module.exports = require("./src/security/permissions")

packages/auth/redis.js

@@ -0,0 +1,4 @@
+module.exports = {
+  Client: require("./src/redis"),
+  utils: require("./src/redis/utils"),
+}

packages/auth/roles.js

@@ -0,0 +1 @@
+module.exports = require("./src/security/roles")

packages/auth/src/db/index.js

@@ -7,3 +7,7 @@ module.exports.setDB = pouch => {
 module.exports.getDB = dbName => {
   return new Pouch(dbName)
 }
+
+module.exports.getCouch = () => {
+  return Pouch
+}

packages/auth/src/db/utils.js

@@ -1,5 +1,6 @@
 const { newid } = require("../hashing")
 const Replication = require("./Replication")
+const { getCouch } = require("./index")
 
 const UNICODE_MAX = "\ufff0"
 const SEPARATOR = "_"
@@ -25,6 +26,7 @@ const DocumentTypes = {
   APP: "app",
   APP_DEV: "app_dev",
   APP_METADATA: "app_metadata",
+  ROLE: "role",
 }
 
 exports.DocumentTypes = DocumentTypes
@@ -32,6 +34,29 @@ exports.APP_PREFIX = DocumentTypes.APP + SEPARATOR
 exports.APP_DEV_PREFIX = DocumentTypes.APP_DEV + SEPARATOR
 exports.SEPARATOR = SEPARATOR
 
+/**
+ * If creating DB allDocs/query params with only a single top level ID this can be used, this
+ * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.
+ * More complex cases such as link docs and rows which have multiple levels of IDs that their
+ * ID consists of need their own functions to build the allDocs parameters.
+ * @param {string} docType The type of document which input params are being built for, e.g. user,
+ * link, app, table and so on.
+ * @param {string|null} docId The ID of the document minus its type - this is only needed if looking
+ * for a singular document.
+ * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
+ * @returns {object} Parameters which can then be used with an allDocs request.
+ */
+function getDocParams(docType, docId = null, otherProps = {}) {
+  if (docId == null) {
+    docId = ""
+  }
+  return {
+    ...otherProps,
+    startkey: `${docType}${SEPARATOR}${docId}`,
+    endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
+  }
+}
+
 /**
  * Generates a new group ID.
  * @returns {string} The new group ID which the group doc can be stored under.
@@ -101,6 +126,51 @@ exports.getTemplateParams = (ownerId, templateId, otherProps = {}) => {
   }
 }
 
+/**
+ * Generates a new role ID.
+ * @returns {string} The new role ID which the role doc can be stored under.
+ */
+exports.generateRoleID = id => {
+  return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}`
+}
+
+/**
+ * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
+ */
+exports.getRoleParams = (roleId = null, otherProps = {}) => {
+  return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
+}
+
+/**
+ * Lots of different points in the system need to find the full list of apps, this will
+ * enumerate the entire CouchDB cluster and get the list of databases (every app).
+ * NOTE: this operation is fine in self hosting, but cannot be used when hosting many
+ * different users/companies apps as there is no security around it - all apps are returned.
+ * @return {Promise<object[]>} returns the app information document stored in each app database.
+ */
+exports.getAllApps = async (devApps = false) => {
+  const CouchDB = getCouch()
+  let allDbs = await CouchDB.allDbs()
+  const appDbNames = allDbs.filter(dbName =>
+    dbName.startsWith(exports.APP_PREFIX)
+  )
+  const appPromises = appDbNames.map(db => new CouchDB(db).get(DocumentTypes.APP_METADATA))
+  if (appPromises.length === 0) {
+    return []
+  } else {
+    const response = await Promise.allSettled(appPromises)
+    const apps = response
+      .filter(result => result.status === "fulfilled")
+      .map(({ value }) => value)
+    return apps.filter(app => {
+      if (devApps) {
+        return app.appId.startsWith(exports.APP_DEV_PREFIX)
+      }
+      return !app.appId.startsWith(exports.APP_DEV_PREFIX)
+    })
+  }
+}
+
 /**
  * Generates a new configuration ID.
  * @returns {string} The new configuration ID which the config doc can be stored under.

packages/auth/src/objectStore/index.js

@@ -126,7 +126,13 @@ exports.makeSureBucketExists = async (client, bucketName) => {
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
  */
-exports.upload = async ({ bucket: bucketName, filename, path, type, metadata }) => {
+exports.upload = async ({
+  bucket: bucketName,
+  filename,
+  path,
+  type,
+  metadata,
+}) => {
   const extension = [...filename.split(".")].pop()
   const fileBytes = fs.readFileSync(path)
 

packages/server/src/utilities/security/roles.js → packages/auth/src/security/roles.js

@@ -1,7 +1,12 @@
-const CouchDB = require("../../db")
+const { getDB } = require("../db")
 const { cloneDeep } = require("lodash/fp")
 const { BUILTIN_PERMISSION_IDS, higherPermission } = require("./permissions")
-const { generateRoleID, DocumentTypes, SEPARATOR } = require("../../db/utils")
+const {
+  generateRoleID,
+  getRoleParams,
+  DocumentTypes,
+  SEPARATOR,
+} = require("../db/utils")
 
 const BUILTIN_IDS = {
   ADMIN: "ADMIN",
@@ -11,6 +16,14 @@ const BUILTIN_IDS = {
   BUILDER: "BUILDER",
 }
 
+// exclude internal roles like builder
+const EXTERNAL_BUILTIN_ROLE_IDS = [
+  BUILTIN_IDS.ADMIN,
+  BUILTIN_IDS.POWER,
+  BUILTIN_IDS.BASIC,
+  BUILTIN_IDS.PUBLIC,
+]
+
 function Role(id, name) {
   this._id = id
   this.name = name
@@ -116,7 +129,7 @@ exports.getRole = async (appId, roleId) => {
     )
   }
   try {
-    const db = new CouchDB(appId)
+    const db = getDB(appId)
     const dbRole = await db.get(exports.getDBRoleID(roleId))
     role = Object.assign(role, dbRole)
     // finalise the ID
@@ -192,6 +205,39 @@ exports.getUserPermissions = async (appId, userRoleId) => {
   }
 }
 
+/**
+ * Given an app ID this will retrieve all of the roles that are currently within that app.
+ * @param {string} appId The ID of the app to retrieve the roles from.
+ * @return {Promise<object[]>} An array of the role objects that were found.
+ */
+exports.getAllRoles = async appId => {
+  const db = getDB(appId)
+  const body = await db.allDocs(
+    getRoleParams(null, {
+      include_docs: true,
+    })
+  )
+  let roles = body.rows.map(row => row.doc)
+  const builtinRoles = exports.getBuiltinRoles()
+
+  // need to combine builtin with any DB record of them (for sake of permissions)
+  for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) {
+    const builtinRole = builtinRoles[builtinRoleId]
+    const dbBuiltin = roles.filter(
+      dbRole => exports.getExternalRoleID(dbRole._id) === builtinRoleId
+    )[0]
+    if (dbBuiltin == null) {
+      roles.push(builtinRole)
+    } else {
+      // remove role and all back after combining with the builtin
+      roles = roles.filter(role => role._id !== dbBuiltin._id)
+      dbBuiltin._id = exports.getExternalRoleID(dbBuiltin._id)
+      roles.push(Object.assign(builtinRole, dbBuiltin))
+    }
+  }
+  return roles
+}
+
 class AccessController {
   constructor(appId) {
     this.appId = appId

packages/builder/src/components/start/AppCard.svelte

@@ -37,8 +37,8 @@
         </MenuItem>
         {#if deletable}
           <MenuItem on:click={() => deleteApp(app)} icon="Delete">
-          Delete
-        </MenuItem>
+            Delete
+          </MenuItem>
         {/if}
         {#if app.lockedBy && app.lockedBy?.email === $auth.user?.email}
           <MenuItem on:click={() => releaseLock(app.appId)} icon="LockOpen">

packages/builder/src/constants/index.js

@@ -11,7 +11,7 @@ export const FrontendTypes = {
 
 export const AppStatus = {
   DEV: "dev",
-  PUBLISHED: "published"
+  PUBLISHED: "published",
 }
 
 // fields on the user table that cannot be edited

packages/server/src/api/controllers/application.js

@@ -20,10 +20,7 @@ const {
   DocumentTypes,
   AppStatus,
 } = require("../../db/utils")
-const {
-  BUILTIN_ROLE_IDS,
-  AccessController,
-} = require("../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS, AccessController } = require("@budibase/auth/roles")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
 const {
   createHomeScreen,
@@ -120,15 +117,8 @@ async function createInstance(template) {
 }
 
 exports.fetch = async function (ctx) {
-  let apps = await getAllApps()
-
-  const isDev = ctx.query.status === AppStatus.DEV
-  apps = apps.filter(app => {
-    if (isDev) {
-      return app.appId.startsWith(DocumentTypes.APP_DEV)
-    }
-    return !app.appId.startsWith(DocumentTypes.APP_DEV)
-  })
+  const isDev = ctx.query && ctx.query.status === AppStatus.DEV
+  const apps = await getAllApps(isDev)
 
   // get the locks for all the dev apps
   if (isDev) {

packages/server/src/api/controllers/deploy/index.js

@@ -1,6 +1,6 @@
 const PouchDB = require("../../../db")
 const Deployment = require("./Deployment")
-const { Replication, StaticDatabases } = require("@budibase/auth").db
+const { Replication, StaticDatabases } = require("@budibase/auth/db")
 const { DocumentTypes } = require("../../../db/utils")
 
 // the max time we can wait for an invalidation to complete before considering it failed

packages/server/src/api/controllers/hosting.js

@@ -1,6 +1,6 @@
 const CouchDB = require("../../db")
 const { getDeployedApps } = require("../../utilities/workerRequests")
-const { getScopedConfig } = require("@budibase/auth").db
+const { getScopedConfig } = require("@budibase/auth/db")
 const { Configs } = require("@budibase/auth").constants
 const { checkSlashesInUrl } = require("../../utilities")
 

packages/server/src/api/controllers/permission.js

@@ -3,19 +3,19 @@ const {
   PermissionLevels,
   isPermissionLevelHigherThanRead,
   higherPermission,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const {
   isBuiltin,
   getDBRoleID,
   getExternalRoleID,
   getBuiltinRoles,
-} = require("../../utilities/security/roles")
+} = require("@budibase/auth/roles")
 const { getRoleParams } = require("../../db/utils")
 const CouchDB = require("../../db")
 const {
   CURRENTLY_SUPPORTED_LEVELS,
   getBasePermissions,
-} = require("../../utilities/security/utilities")
+} = require("../../utilities/security")
 
 const PermissionUpdateType = {
   REMOVE: "remove",

packages/server/src/api/controllers/role.js

@@ -1,12 +1,12 @@
 const CouchDB = require("../../db")
 const {
   getBuiltinRoles,
-  BUILTIN_ROLE_IDS,
   Role,
   getRole,
   isBuiltin,
   getExternalRoleID,
-} = require("../../utilities/security/roles")
+  getAllRoles,
+} = require("@budibase/auth/roles")
 const {
   generateRoleID,
   getRoleParams,
@@ -19,14 +19,6 @@ const UpdateRolesOptions = {
   REMOVED: "removed",
 }
 
-// exclude internal roles like builder
-const EXTERNAL_BUILTIN_ROLE_IDS = [
-  BUILTIN_ROLE_IDS.ADMIN,
-  BUILTIN_ROLE_IDS.POWER,
-  BUILTIN_ROLE_IDS.BASIC,
-  BUILTIN_ROLE_IDS.PUBLIC,
-]
-
 async function updateRolesOnUserTable(db, roleId, updateOption) {
   const table = await db.get(InternalTables.USER_METADATA)
   const schema = table.schema
@@ -51,31 +43,7 @@ async function updateRolesOnUserTable(db, roleId, updateOption) {
 }
 
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
-  const body = await db.allDocs(
-    getRoleParams(null, {
-      include_docs: true,
-    })
-  )
-  let roles = body.rows.map(row => row.doc)
-  const builtinRoles = getBuiltinRoles()
-
-  // need to combine builtin with any DB record of them (for sake of permissions)
-  for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) {
-    const builtinRole = builtinRoles[builtinRoleId]
-    const dbBuiltin = roles.filter(
-      dbRole => getExternalRoleID(dbRole._id) === builtinRoleId
-    )[0]
-    if (dbBuiltin == null) {
-      roles.push(builtinRole)
-    } else {
-      // remove role and all back after combining with the builtin
-      roles = roles.filter(role => role._id !== dbBuiltin._id)
-      dbBuiltin._id = getExternalRoleID(dbBuiltin._id)
-      roles.push(Object.assign(builtinRole, dbBuiltin))
-    }
-  }
-  ctx.body = roles
+  ctx.body = await getAllRoles(ctx.appId)
 }
 
 exports.find = async function (ctx) {

packages/server/src/api/controllers/routing.js

@@ -2,7 +2,7 @@ const { getRoutingInfo } = require("../../utilities/routing")
 const {
   getUserRoleHierarchy,
   BUILTIN_ROLE_IDS,
-} = require("../../utilities/security/roles")
+} = require("@budibase/auth/roles")
 
 const URL_SEPARATOR = "/"
 

packages/server/src/api/controllers/screen.js

@@ -1,6 +1,6 @@
 const CouchDB = require("../../db")
 const { getScreenParams, generateScreenID } = require("../../db/utils")
-const { AccessController } = require("../../utilities/security/roles")
+const { AccessController } = require("@budibase/auth/roles")
 
 exports.fetch = async ctx => {
   const appId = ctx.appId

packages/server/src/api/controllers/user.js

@@ -5,7 +5,7 @@ const {
   getGlobalIDFromUserMetadataID,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
-const { getRole, BUILTIN_ROLE_IDS } = require("../../utilities/security/roles")
+const { getRole, BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const {
   getGlobalUsers,
   saveGlobalUser,

packages/server/src/api/routes/analytics.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
 const controller = require("../controllers/analytics")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/apikeys.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/apikeys")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/application.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/application")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/automation.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const Joi = require("joi")
 const { bodyResource, paramResource } = require("../../middleware/resourceId")
 

packages/server/src/api/routes/backup.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/backup")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/component.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/component")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/datasource.js

@@ -5,7 +5,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/deploy.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/deploy")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/dev.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/dev")
 const env = require("../../environment")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/hosting.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/hosting")
 const authorized = require("../../middleware/authorized")
 const selfhost = require("../../middleware/selfhost")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/integration.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/integration")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/layout.js

@@ -1,6 +1,6 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const controller = require("../controllers/layout")
 
 const router = Router()

packages/server/src/api/routes/permission.js

@@ -1,10 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/permission")
 const authorized = require("../../middleware/authorized")
-const {
-  BUILDER,
-  PermissionLevels,
-} = require("../../utilities/security/permissions")
+const { BUILDER, PermissionLevels } = require("@budibase/auth/permissions")
 const Joi = require("joi")
 const joiValidator = require("../../middleware/joi-validator")
 

packages/server/src/api/routes/query.js

@@ -1,12 +1,12 @@
 const Router = require("@koa/router")
 const queryController = require("../controllers/query")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
 const Joi = require("joi")
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+  BUILDER,
+} = require("@budibase/auth/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const {
   bodyResource,

packages/server/src/api/routes/role.js

@@ -1,15 +1,13 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/role")
 const authorized = require("../../middleware/authorized")
-const {
-  BUILDER,
-  PermissionLevels,
-} = require("../../utilities/security/permissions")
 const Joi = require("joi")
 const joiValidator = require("../../middleware/joi-validator")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("../../utilities/security/permissions")
+  BUILDER,
+  PermissionLevels,
+} = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/routing.js

@@ -1,6 +1,6 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const controller = require("../controllers/routing")
 
 const router = Router()

packages/server/src/api/routes/row.js

@@ -9,7 +9,7 @@ const {
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/screen.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/screen")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const Joi = require("joi")
 

packages/server/src/api/routes/script.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/script")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/search.js

@@ -3,7 +3,7 @@ const controller = require("../controllers/search")
 const {
   PermissionTypes,
   PermissionLevels,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const authorized = require("../../middleware/authorized")
 const { paramResource } = require("../../middleware/resourceId")
 

packages/server/src/api/routes/static.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const usage = require("../../middleware/usageQuota")
 const env = require("../../environment")
 

packages/server/src/api/routes/table.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const Joi = require("joi")
 

packages/server/src/api/routes/templates.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/templates")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/tests/application.spec.js

@@ -1,6 +1,14 @@
 const { clearAllApps, checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 
+jest.mock("../../../utilities/redis", () => ({
+  init: jest.fn(),
+  getAllLocks: () => {
+    return []
+  },
+  updateLock: jest.fn(),
+}))
+
 describe("/applications", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -40,7 +48,7 @@ describe("/applications", () => {
       await config.createApp(request, "app2")
 
       const res = await request
-        .get("/api/applications")
+        .get("/api/applications?status=dev")
         .set(config.defaultHeaders())
         .expect('Content-Type', /json/)
         .expect(200)

packages/server/src/api/routes/tests/permissions.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
 

packages/server/src/api/routes/tests/role.spec.js

@@ -1,7 +1,7 @@
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("../../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const setup = require("./utilities")
 const { basicRole } = setup.structures
 

packages/server/src/api/routes/tests/routing.spec.js

@@ -1,7 +1,7 @@
 const setup = require("./utilities")
 const { basicScreen } = setup.structures
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const workerRequests = require("../../../utilities/workerRequests")
 
 const route = "/test"

packages/server/src/api/routes/tests/user.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 const { basicUser } = setup.structures

packages/server/src/api/routes/tests/utilities/TestFunctions.js

@@ -14,7 +14,7 @@ exports.getAllTableRows = async config => {
 }
 
 exports.clearAllApps = async () => {
-  const req = {}
+  const req = { query: { status: "dev" } }
   await appController.fetch(req)
   const apps = req.body
   if (!apps || apps.length <= 0) {

packages/server/src/api/routes/user.js

@@ -4,7 +4,7 @@ const authorized = require("../../middleware/authorized")
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const usage = require("../../middleware/usageQuota")
 
 const router = Router()

packages/server/src/api/routes/view.js

@@ -7,7 +7,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const usage = require("../../middleware/usageQuota")
 
 const router = Router()

packages/server/src/api/routes/webhook.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/webhook")
 const authorized = require("../../middleware/authorized")
 const joiValidator = require("../../middleware/joi-validator")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const Joi = require("joi")
 
 const router = Router()

packages/server/src/automations/steps/createUser.js

@@ -1,4 +1,4 @@
-const roles = require("../../utilities/security/roles")
+const roles = require("@budibase/auth/roles")
 const userController = require("../../api/controllers/user")
 const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")

packages/server/src/automations/tests/createUser.spec.js

@@ -1,6 +1,6 @@
 const usageQuota = require("../../utilities/usageQuota")
 const setup = require("./utilities")
-const { BUILTIN_ROLE_IDS } = require("../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { InternalTables } = require("../../db/utils")
 
 jest.mock("../../utilities/usageQuota")

packages/server/src/automations/triggers.js

@@ -6,7 +6,7 @@ const Queue = env.isTest()
   : require("bull")
 const { getAutomationParams } = require("../db/utils")
 const { coerce } = require("../utilities/rowProcessor")
-const { utils } = require("@budibase/auth").redis
+const { utils } = require("@budibase/auth/redis")
 
 const { opts } = utils.getRedisOptions()
 let automationQueue = new Queue("automationQueue", { redis: opts })

packages/server/src/constants/index.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { UserStatus } = require("@budibase/auth").constants
 const { ObjectStoreBuckets } = require("@budibase/auth").objectStore
 

packages/server/src/constants/screens.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { BASE_LAYOUT_PROP_IDS } = require("./layouts")
 const { LOGO_URL } = require("../constants")
 

packages/server/src/db/utils.js

@@ -1,10 +1,12 @@
 const newid = require("./newid")
 const {
   DocumentTypes: CoreDocTypes,
+  getRoleParams,
+  generateRoleID,
   APP_DEV_PREFIX,
   APP_PREFIX,
   SEPARATOR,
-} = require("@budibase/auth").db
+} = require("@budibase/auth/db")
 
 const UNICODE_MAX = "\ufff0"
 
@@ -24,12 +26,12 @@ const DocumentTypes = {
   APP: CoreDocTypes.APP,
   APP_DEV: CoreDocTypes.APP_DEV,
   APP_METADATA: CoreDocTypes.APP_METADATA,
+  ROLE: CoreDocTypes.ROLE,
   TABLE: "ta",
   ROW: "ro",
   USER: "us",
   AUTOMATION: "au",
   LINK: "li",
-  ROLE: "role",
   WEBHOOK: "wh",
   INSTANCE: "inst",
   LAYOUT: "layout",
@@ -62,6 +64,9 @@ exports.UNICODE_MAX = UNICODE_MAX
 exports.SearchIndexes = SearchIndexes
 exports.AppStatus = AppStatus
 
+exports.generateRoleID = generateRoleID
+exports.getRoleParams = getRoleParams
+
 exports.getQueryIndex = viewName => {
   return `database/${viewName}`
 }
@@ -225,21 +230,6 @@ exports.generateDevAppID = appId => {
   return `${DocumentTypes.APP_DEV}${SEPARATOR}${uuid}`
 }
 
-/**
- * Generates a new role ID.
- * @returns {string} The new role ID which the role doc can be stored under.
- */
-exports.generateRoleID = id => {
-  return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}`
-}
-
-/**
- * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
- */
-exports.getRoleParams = (roleId = null, otherProps = {}) => {
-  return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
-}
-
 /**
  * Generates a new layout ID.
  * @returns {string} The new layout ID which the layout doc can be stored under.

packages/server/src/middleware/authorized.js

@@ -1,9 +1,9 @@
-const { getUserPermissions } = require("../utilities/security/roles")
+const { getUserPermissions } = require("@budibase/auth/roles")
 const {
   PermissionTypes,
   doesHaveResourcePermission,
   doesHaveBasePermission,
-} = require("../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const { APP_DEV_PREFIX } = require("../db/utils")
 const { doesUserHaveLock, updateLock } = require("../utilities/redis")
 

packages/server/src/middleware/currentapp.js

@@ -1,8 +1,8 @@
 const { getAppId, setCookie, getCookie } = require("@budibase/auth").utils
 const { Cookies } = require("@budibase/auth").constants
-const { getRole } = require("../utilities/security/roles")
+const { getRole } = require("@budibase/auth/roles")
 const { getGlobalUsers } = require("../utilities/workerRequests")
-const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { generateUserMetadataID } = require("../db/utils")
 
 module.exports = async (ctx, next) => {

packages/server/src/middleware/tests/authorized.spec.js

@@ -1,6 +1,6 @@
 const authorizedMiddleware = require("../authorized")
 const env = require("../../environment")
-const { PermissionTypes, PermissionLevels } = require("../../utilities/security/permissions")
+const { PermissionTypes, PermissionLevels } = require("@budibase/auth/permissions")
 jest.mock("../../environment", () => ({
     prod: false,
     isTest: () => true,

packages/server/src/middleware/tests/currentapp.spec.js

@@ -8,7 +8,8 @@ function mockWorker() {
         _id: "us_uuid1",
         roles: {
           "app_test": "BASIC",
-        }
+        },
+        roleId: "BASIC",
       }
     }
   }))

packages/server/src/middleware/tests/selfhost.spec.js

@@ -1,7 +1,6 @@
 const selfHostMiddleware = require("../selfhost")
 const env = require("../../environment")
-jest.mock("../../environment") 
-jest.mock("../../utilities/builder/hosting") 
+jest.mock("../../environment")
 
 class TestConfiguration {
   constructor() {

packages/server/src/tests/utilities/TestConfiguration.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const env = require("../../environment")
 const {
   basicTable,
@@ -16,7 +16,7 @@ const supertest = require("supertest")
 const { cleanup } = require("../../utilities/fileSystem")
 const { Cookies } = require("@budibase/auth").constants
 const { jwt } = require("@budibase/auth").auth
-const { StaticDatabases } = require("@budibase/auth").db
+const { StaticDatabases } = require("@budibase/auth/db")
 const CouchDB = require("../../db")
 
 const GLOBAL_USER_ID = "us_uuid1"

packages/server/src/tests/utilities/structures.js

@@ -1,7 +1,5 @@
-const { BUILTIN_ROLE_IDS } = require("../../utilities/security/roles")
-const {
-  BUILTIN_PERMISSION_IDS,
-} = require("../../utilities/security/permissions")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_PERMISSION_IDS } = require("@budibase/auth/permissions")
 const { createHomeScreen } = require("../../constants/screens")
 const { EMPTY_LAYOUT } = require("../../constants/layouts")
 const { cloneDeep } = require("lodash/fp")

packages/server/src/utilities/index.js

@@ -1,36 +1,13 @@
 const env = require("../environment")
-const { APP_PREFIX, DocumentTypes } = require("../db/utils")
-const CouchDB = require("../db")
 const { OBJ_STORE_DIRECTORY, ObjectStoreBuckets } = require("../constants")
+const { getAllApps } = require("@budibase/auth/db")
 
 const BB_CDN = "https://cdn.app.budi.live/assets"
 
 exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))
 
 exports.isDev = env.isDev
-
-/**
- * Lots of different points in the app need to find the full list of apps, this will
- * enumerate the entire CouchDB cluster and get the list of databases (every app).
- * NOTE: this operation is fine in self hosting, but cannot be used when hosting many
- * different users/companies apps as there is no security around it - all apps are returned.
- * @return {Promise<object[]>} returns the app information document stored in each app database.
- */
-exports.getAllApps = async () => {
-  let allDbs = await CouchDB.allDbs()
-  const appDbNames = allDbs.filter(dbName => dbName.startsWith(APP_PREFIX))
-  const appPromises = appDbNames.map(db =>
-    new CouchDB(db).get(DocumentTypes.APP_METADATA)
-  )
-  if (appPromises.length === 0) {
-    return []
-  } else {
-    const response = await Promise.allSettled(appPromises)
-    return response
-      .filter(result => result.status === "fulfilled")
-      .map(({ value }) => value)
-  }
-}
+exports.getAllApps = getAllApps
 
 /**
  * Makes sure that a URL has the correct number of slashes, while maintaining the

packages/server/src/utilities/redis.js

@@ -1,4 +1,4 @@
-const { Client, utils } = require("@budibase/auth").redis
+const { Client, utils } = require("@budibase/auth/redis")
 const { getGlobalIDFromUserMetadataID } = require("../db/utils")
 
 const APP_DEV_LOCK_SECONDS = 600

packages/server/src/utilities/security/utilities.js → packages/server/src/utilities/security.js

@@ -3,12 +3,9 @@ const {
   PermissionTypes,
   getBuiltinPermissionByID,
   isPermissionLevelHigherThanRead,
-} = require("../../utilities/security/permissions")
-const {
-  lowerBuiltinRoleID,
-  getBuiltinRoles,
-} = require("../../utilities/security/roles")
-const { DocumentTypes } = require("../../db/utils")
+} = require("@budibase/auth/permissions")
+const { lowerBuiltinRoleID, getBuiltinRoles } = require("@budibase/auth/roles")
+const { DocumentTypes } = require("../db/utils")
 
 const CURRENTLY_SUPPORTED_LEVELS = [
   PermissionLevels.WRITE,

packages/server/src/utilities/workerRequests.js

@@ -1,7 +1,7 @@
 const fetch = require("node-fetch")
 const env = require("../environment")
 const { checkSlashesInUrl } = require("./index")
-const { BUILTIN_ROLE_IDS } = require("./security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 
 function getAppRole(appId, user) {
   if (!user.roles) {

packages/worker/src/api/controllers/admin/roles.js

@@ -0,0 +1,24 @@
+const { getAllRoles } = require("@budibase/auth/roles")
+const { getAllApps } = require("@budibase/auth/db")
+
+exports.fetch = async ctx => {
+  // always use the dev apps as they'll be most up to date (true)
+  const apps = await getAllApps(true)
+  const promises = []
+  for (let app of apps) {
+    promises.push(getAllRoles(app._id))
+  }
+  const roles = await Promise.all(promises)
+  const response = {}
+  for (let app of apps) {
+    response[app._id] = roles.shift()
+  }
+  ctx.body = response
+}
+
+exports.find = async ctx => {
+  const appId = ctx.params.appId
+  ctx.body = {
+    roles: await getAllRoles(appId),
+  }
+}

packages/worker/src/api/routes/admin/roles.js

@@ -0,0 +1,10 @@
+const Router = require("@koa/router")
+const controller = require("../../controllers/admin/roles")
+
+const router = Router()
+
+router
+  .get("/api/admin/roles", controller.fetch)
+  .get("/api/admin/roles/:appId", controller.find)
+
+module.exports = router

packages/worker/src/api/routes/admin/users.js

@@ -45,6 +45,7 @@ router
   .post("/api/admin/users/init", controller.adminUser)
   .delete("/api/admin/users/:id", controller.destroy)
   .get("/api/admin/users/:id", controller.find)
+  .get("/api/admin/roles/:appId")
   .post("/api/admin/users/invite", buildInviteValidation(), controller.invite)
   .post(
     "/api/admin/users/invite/accept",

packages/worker/src/api/routes/index.js

@@ -4,6 +4,7 @@ const groupRoutes = require("./admin/groups")
 const templateRoutes = require("./admin/templates")
 const emailRoutes = require("./admin/email")
 const authRoutes = require("./admin/auth")
+const roleRoutes = require("./admin/roles")
 const appRoutes = require("./app")
 
 exports.routes = [
@@ -14,4 +15,5 @@ exports.routes = [
   appRoutes,
   templateRoutes,
   emailRoutes,
+  roleRoutes,
 ]

packages/worker/src/utilities/redis.js

@@ -1,4 +1,4 @@
-const { Client, utils } = require("@budibase/auth").redis
+const { Client, utils } = require("@budibase/auth/redis")
 const { newid } = require("@budibase/auth").utils
 
 function getExpirySecondsForDB(db) {