Merge pull request #6515 from Budibase/feature/user-pagination

User pagination

packages/backend-core/src/db/constants.js

@@ -1,4 +1,5 @@
 exports.SEPARATOR = "_"
+exports.UNICODE_MAX = "\ufff0"
 
 const PRE_APP = "app"
 const PRE_DEV = "dev"

packages/backend-core/src/db/utils.ts

@@ -1,7 +1,7 @@
 import { newid } from "../hashing"
 import { DEFAULT_TENANT_ID, Configs } from "../constants"
 import env from "../environment"
-import { SEPARATOR, DocumentTypes } from "./constants"
+import { SEPARATOR, DocumentTypes, UNICODE_MAX } from "./constants"
 import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy"
 import fetch from "node-fetch"
 import { doWithDB, allDbs } from "./index"
@@ -12,8 +12,6 @@ import { isDevApp, isDevAppID } from "./conversions"
 import { APP_PREFIX } from "./constants"
 import * as events from "../events"
 
-const UNICODE_MAX = "\ufff0"
-
 export const ViewNames = {
   USER_BY_EMAIL: "by_email",
   BY_API_KEY: "by_api_key",
@@ -93,13 +91,17 @@ export function generateGlobalUserID(id?: any) {
 /**
  * Gets parameters for retrieving users.
  */
-export function getGlobalUserParams(globalId: any, otherProps = {}) {
+export function getGlobalUserParams(globalId: any, otherProps: any = {}) {
   if (!globalId) {
     globalId = ""
   }
+  const startkey = otherProps?.startkey
   return {
     ...otherProps,
-    startkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}`,
+    // need to include this incase pagination
+    startkey: startkey
+      ? startkey
+      : `${DocumentTypes.USER}${SEPARATOR}${globalId}`,
     endkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}${UNICODE_MAX}`,
   }
 }
@@ -434,6 +436,26 @@ export const getPlatformUrl = async (opts = { tenantAware: true }) => {
   return platformUrl
 }
 
+export function pagination(
+  data: any[],
+  pageSize: number,
+  { paginate, property } = { paginate: true, property: "_id" }
+) {
+  if (!paginate) {
+    return { data, hasNextPage: false }
+  }
+  const hasNextPage = data.length > pageSize
+  let nextPage = undefined
+  if (hasNextPage) {
+    nextPage = property ? data[pageSize]?.[property] : data[pageSize]?._id
+  }
+  return {
+    data: data.slice(0, pageSize),
+    hasNextPage,
+    nextPage,
+  }
+}
+
 export async function getScopedConfig(db: any, params: any) {
   const configDoc = await getScopedFullConfig(db, params)
   return configDoc && configDoc.config ? configDoc.config : configDoc

packages/backend-core/src/users.js

@@ -1,5 +1,6 @@
 const { ViewNames } = require("./db/utils")
 const { queryGlobalView } = require("./db/views")
+const { UNICODE_MAX } = require("./db/constants")
 
 /**
  * Given an email address this will use a view to search through
@@ -19,3 +20,24 @@ exports.getGlobalUserByEmail = async email => {
 
   return response
 }
+
+/**
+ * Performs a starts with search on the global email view.
+ */
+exports.searchGlobalUsersByEmail = async (email, opts) => {
+  if (typeof email !== "string") {
+    throw new Error("Must provide a string to search by")
+  }
+  const lcEmail = email.toLowerCase()
+  // handle if passing up startkey for pagination
+  const startkey = opts && opts.startkey ? opts.startkey : lcEmail
+  let response = await queryGlobalView(ViewNames.USER_BY_EMAIL, {
+    ...opts,
+    startkey,
+    endkey: `${lcEmail}${UNICODE_MAX}`,
+  })
+  if (!response) {
+    response = []
+  }
+  return Array.isArray(response) ? response : [response]
+}

packages/builder/src/helpers/pagination.js

@@ -0,0 +1,66 @@
+import { writable } from "svelte/store"
+
+function defaultValue() {
+  return {
+    nextPage: null,
+    page: undefined,
+    hasPrevPage: false,
+    hasNextPage: false,
+    loading: false,
+    pageNumber: 1,
+    pages: [],
+  }
+}
+
+export function createPaginationStore() {
+  const { subscribe, set, update } = writable(defaultValue())
+
+  function prevPage() {
+    update(state => {
+      state.pageNumber--
+      state.nextPage = state.pages.pop()
+      state.page = state.pages[state.pages.length - 1]
+      state.hasPrevPage = state.pageNumber > 1
+      return state
+    })
+  }
+
+  function nextPage() {
+    update(state => {
+      state.pageNumber++
+      state.page = state.nextPage
+      state.pages.push(state.page)
+      state.hasPrevPage = state.pageNumber > 1
+      return state
+    })
+  }
+
+  function fetched(hasNextPage, nextPage) {
+    update(state => {
+      state.hasNextPage = hasNextPage
+      state.nextPage = nextPage
+      state.loading = false
+      return state
+    })
+  }
+
+  function loading(loading = true) {
+    update(state => {
+      state.loading = loading
+      return state
+    })
+  }
+
+  function reset() {
+    set(defaultValue())
+  }
+
+  return {
+    subscribe,
+    prevPage,
+    nextPage,
+    fetched,
+    loading,
+    reset,
+  }
+}

packages/builder/src/pages/builder/portal/manage/users/_components/AddUserModal.svelte

@@ -10,7 +10,9 @@
   } from "@budibase/bbui"
   import { createValidationStore, emailValidator } from "helpers/validation"
   import { users } from "stores/portal"
+  import { createEventDispatcher } from "svelte"
 
+  const dispatch = createEventDispatcher()
   const password = Math.random().toString(36).substring(2, 22)
   const options = ["Email onboarding", "Basic onboarding"]
   const [email, error, touched] = createValidationStore("", emailValidator)
@@ -39,6 +41,7 @@
         forceResetPassword: true,
       })
       notifications.success("Successfully created user")
+      dispatch("created")
     } catch (error) {
       notifications.error("Error creating user")
     }

packages/builder/src/pages/builder/portal/manage/users/index.svelte

@@ -12,41 +12,36 @@
     Layout,
     Modal,
     notifications,
+    Pagination,
   } from "@budibase/bbui"
   import TagsRenderer from "./_components/TagsTableRenderer.svelte"
   import AddUserModal from "./_components/AddUserModal.svelte"
   import { users } from "stores/portal"
-  import { onMount } from "svelte"
+  import { createPaginationStore } from "helpers/pagination"
 
   const schema = {
     email: {},
     developmentAccess: { displayName: "Development Access", type: "boolean" },
     adminAccess: { displayName: "Admin Access", type: "boolean" },
-    // role: { type: "options" },
     group: {},
-    // access: {},
-    // group: {}
   }
 
-  let search
+  let pageInfo = createPaginationStore()
-  $: filteredUsers = $users
+  let search = undefined
-    .filter(user => user.email.includes(search || ""))
+  $: page = $pageInfo.page
-    .map(user => ({
+  $: fetchUsers(page, search)
-      ...user,
-      group: ["All users"],
-      developmentAccess: !!user.builder?.global,
-      adminAccess: !!user.admin?.global,
-    }))
 
   let createUserModal
 
-  onMount(async () => {
+  async function fetchUsers(page, search) {
     try {
-      await users.init()
+      pageInfo.loading()
+      await users.search({ page, search })
+      pageInfo.fetched($users.hasNextPage, $users.nextPage)
     } catch (error) {
       notifications.error("Error getting user list")
     }
-  })
+  }
 </script>
 
 <Layout noPadding>
@@ -75,17 +70,31 @@
     <Table
       on:click={({ detail }) => $goto(`./${detail._id}`)}
       {schema}
-      data={filteredUsers || $users}
+      data={$users.data}
       allowEditColumns={false}
       allowEditRows={false}
       allowSelectRows={false}
       customRenderers={[{ column: "group", component: TagsRenderer }]}
     />
+    <div class="pagination">
+      <Pagination
+        page={$pageInfo.pageNumber}
+        hasPrevPage={$pageInfo.loading ? false : $pageInfo.hasPrevPage}
+        hasNextPage={$pageInfo.loading ? false : $pageInfo.hasNextPage}
+        goToPrevPage={pageInfo.prevPage}
+        goToNextPage={pageInfo.nextPage}
+      />
+    </div>
   </Layout>
 </Layout>
 
 <Modal bind:this={createUserModal}>
-  <AddUserModal />
+  <AddUserModal
+    on:created={async () => {
+      pageInfo.reset()
+      await fetchUsers()
+    }}
+  />
 </Modal>
 
 <style>

packages/builder/src/pages/builder/portal/overview/_components/OverviewTab.svelte

@@ -1,14 +1,7 @@
 <script>
   import DashCard from "components/common/DashCard.svelte"
   import { AppStatus } from "constants"
-  import {
+  import { Icon, Heading, Link, Avatar, Layout } from "@budibase/bbui"
-    Icon,
-    Heading,
-    Link,
-    Avatar,
-    notifications,
-    Layout,
-  } from "@budibase/bbui"
   import { store } from "builderStore"
   import clientPackage from "@budibase/client/package.json"
   import { processStringSync } from "@budibase/string-templates"
@@ -20,29 +13,22 @@
   export let navigateTab
   const dispatch = createEventDispatcher()
 
-  const userInit = async () => {
-    try {
-      await users.init()
-    } catch (error) {
-      notifications.error("Error getting user list")
-    }
-  }
-
   const unpublishApp = () => {
     dispatch("unpublish", app)
   }
 
-  let userPromise = userInit()
+  let appEditor, appEditorPromise
 
   $: updateAvailable = clientPackage.version !== $store.version
   $: isPublished = app && app?.status === AppStatus.DEPLOYED
   $: appEditorId = !app?.updatedBy ? $auth.user._id : app?.updatedBy
   $: appEditorText = appEditor?.firstName || appEditor?.email
-  $: filteredUsers = !appEditorId
+  $: fetchAppEditor(appEditorId)
-    ? []
-    : $users.filter(user => user._id === appEditorId)
 
-  $: appEditor = filteredUsers.length ? filteredUsers[0] : null
+  async function fetchAppEditor(editorId) {
+    appEditorPromise = users.get(editorId)
+    appEditor = await appEditorPromise
+  }
 
   const getInitials = user => {
     let initials = ""
@@ -90,7 +76,7 @@
       </DashCard>
       <DashCard title={"Last Edited"} dataCy={"edited-by"}>
         <div class="last-edited-content">
-          {#await userPromise}
+          {#await appEditorPromise}
             <Avatar size="M" initials={"-"} />
           {:then _}
             <div class="updated-by">

packages/builder/src/stores/portal/users.js

@@ -3,11 +3,24 @@ import { API } from "api"
 import { update } from "lodash"
 
 export function createUsersStore() {
-  const { subscribe, set } = writable([])
+  const { subscribe, set } = writable({})
 
-  async function init() {
+  // opts can contain page and search params
-    const users = await API.getUsers()
+  async function search(opts = {}) {
-    set(users)
+    const paged = await API.searchUsers(opts)
+    set({
+      ...paged,
+      ...opts,
+    })
+    return paged
+  }
+
+  async function get(userId) {
+    try {
+      return await API.getUser(userId)
+    } catch (err) {
+      return null
+    }
   }
 
   async function invite({ email, builder, admin }) {
@@ -47,7 +60,8 @@ export function createUsersStore() {
       body.admin = { global: true }
     }
     await API.saveUser(body)
-    await init()
+    // re-search from first page
+    await search()
   }
 
   async function del(id) {
@@ -61,7 +75,8 @@ export function createUsersStore() {
 
   return {
     subscribe,
-    init,
+    search,
+    get,
     invite,
     acceptInvite,
     create,

packages/frontend-core/src/api/user.js

@@ -8,6 +8,34 @@ export const buildUserEndpoints = API => ({
     })
   },
 
+  /**
+   * Gets a list of users in the current tenant.
+   * @param {string} page The page to retrieve
+   * @param {string} search The starts with string to search username/email by.
+   */
+  searchUsers: async ({ page, search } = {}) => {
+    const opts = {}
+    if (page) {
+      opts.page = page
+    }
+    if (search) {
+      opts.search = search
+    }
+    return await API.post({
+      url: `/api/global/users/search`,
+      body: opts,
+    })
+  },
+
+  /**
+   * Get a single user by ID.
+   */
+  getUser: async userId => {
+    return await API.get({
+      url: `/api/global/users/${userId}`,
+    })
+  },
+
   /**
    * Creates a user for an app.
    * @param user the user to create

packages/worker/src/api/controllers/global/configs.js

@@ -1,7 +1,6 @@
 const {
   generateConfigID,
   getConfigParams,
-  getGlobalUserParams,
   getScopedFullConfig,
   getAllApps,
 } = require("@budibase/backend-core/db")
@@ -20,6 +19,7 @@ const {
   bustCache,
 } = require("@budibase/backend-core/cache")
 const { events } = require("@budibase/backend-core")
+const { checkAnyUserExists } = require("../../../utilities/users")
 
 const BB_TENANT_CDN = "https://tenants.cdn.budi.live"
 
@@ -405,12 +405,7 @@ exports.configChecklist = async function (ctx) {
         })
 
         // They have set up an global user
-        const users = await db.allDocs(
+        const userExists = await checkAnyUserExists()
-          getGlobalUserParams(null, {
-            include_docs: true,
-            limit: 1,
-          })
-        )
         return {
           apps: {
             checked: apps.length > 0,
@@ -423,7 +418,7 @@ exports.configChecklist = async function (ctx) {
             link: "/builder/portal/manage/email",
           },
           adminUser: {
-            checked: users && users.rows.length >= 1,
+            checked: userExists,
             label: "Create your first user",
             link: "/builder/portal/manage/users",
           },

packages/worker/src/api/controllers/global/roles.js

@@ -7,7 +7,7 @@ const {
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 const { user: userCache } = require("@budibase/backend-core/cache")
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { users } = require("../../../sdk")
+const { allUsers } = require("../../../sdk/users")
 
 exports.fetch = async ctx => {
   const tenantId = ctx.user.tenantId
@@ -49,10 +49,10 @@ exports.find = async ctx => {
 exports.removeAppRole = async ctx => {
   const { appId } = ctx.params
   const db = getGlobalDB()
-  const allUsers = await users.allUsers(ctx)
+  const users = await allUsers(ctx)
   const bulk = []
   const cacheInvalidations = []
-  for (let user of allUsers) {
+  for (let user of users) {
     if (user.roles[appId]) {
       cacheInvalidations.push(userCache.invalidateUser(user._id))
       delete user.roles[appId]

packages/worker/src/api/controllers/global/users.ts

@@ -8,16 +8,15 @@ import {
   events,
   errors,
   accounts,
-  db as dbUtils,
   users as usersCore,
   tenancy,
   cache,
 } from "@budibase/backend-core"
+import { checkAnyUserExists } from "../../../utilities/users"
 
 export const save = async (ctx: any) => {
   try {
-    const user = await users.save(ctx.request.body)
+    ctx.body = await users.save(ctx.request.body)
-    ctx.body = user
   } catch (err: any) {
     ctx.throw(err.status || 400, err)
   }
@@ -39,15 +38,8 @@ export const adminUser = async (ctx: any) => {
       ctx.throw(403, "Organisation already exists.")
     }
 
-    const response = await tenancy.doWithGlobalDB(tenantId, async (db: any) => {
+    const userExists = await checkAnyUserExists()
-      return db.allDocs(
+    if (userExists) {
-        dbUtils.getGlobalUserParams(null, {
-          include_docs: true,
-        })
-      )
-    })
-
-    if (response.rows.some((row: any) => row.doc.admin)) {
       ctx.throw(
         403,
         "You cannot initialise once an global user has been created."
@@ -96,6 +88,17 @@ export const destroy = async (ctx: any) => {
   }
 }
 
+export const search = async (ctx: any) => {
+  const paginated = await users.paginatedUsers(ctx.request.body)
+  // user hashed password shouldn't ever be returned
+  for (let user of paginated.data) {
+    if (user) {
+      delete user.password
+    }
+  }
+  ctx.body = paginated
+}
+
 // called internally by app server user fetch
 export const fetch = async (ctx: any) => {
   const all = await users.allUsers()

packages/worker/src/api/routes/global/users.js

@@ -46,6 +46,7 @@ router
     controller.save
   )
   .get("/api/global/users", builderOrAdmin, controller.fetch)
+  .post("/api/global/users/search", builderOrAdmin, controller.search)
   .delete("/api/global/users/:id", adminOnly, controller.destroy)
   .get("/api/global/roles/:appId")
   .post(

packages/worker/src/sdk/users/users.ts

@@ -17,9 +17,8 @@ import {
 } from "@budibase/backend-core"
 import { MigrationType } from "@budibase/types"
 
-/**
+const PAGE_LIMIT = 8
- * Retrieves all users from the current tenancy.
+
- */
 export const allUsers = async () => {
   const db = tenancy.getGlobalDB()
   const response = await db.allDocs(
@@ -30,6 +29,37 @@ export const allUsers = async () => {
   return response.rows.map((row: any) => row.doc)
 }
 
+export const paginatedUsers = async ({
+  page,
+  search,
+}: { page?: string; search?: string } = {}) => {
+  const db = tenancy.getGlobalDB()
+  // get one extra document, to have the next page
+  const opts: any = {
+    include_docs: true,
+    limit: PAGE_LIMIT + 1,
+  }
+  // add a startkey if the page was specified (anchor)
+  if (page) {
+    opts.startkey = page
+  }
+  // property specifies what to use for the page/anchor
+  let userList, property
+  // no search, query allDocs
+  if (!search) {
+    const response = await db.allDocs(dbUtils.getGlobalUserParams(null, opts))
+    userList = response.rows.map((row: any) => row.doc)
+    property = "_id"
+  } else {
+    userList = await usersCore.searchGlobalUsersByEmail(search, opts)
+    property = "email"
+  }
+  return dbUtils.pagination(userList, PAGE_LIMIT, {
+    paginate: true,
+    property,
+  })
+}
+
 /**
  * Gets a user by ID from the global database, based on the current tenancy.
  */

packages/worker/src/utilities/users.js

@@ -0,0 +1,17 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getGlobalUserParams } = require("@budibase/backend-core/db")
+
+exports.checkAnyUserExists = async () => {
+  try {
+    const db = getGlobalDB()
+    const users = await db.allDocs(
+      getGlobalUserParams(null, {
+        include_docs: true,
+        limit: 1,
+      })
+    )
+    return users && users.rows.length >= 1
+  } catch (err) {
+    throw new Error("Unable to retrieve user list")
+  }
+}