app and account properties, add account details to all user and tenant identities

packages/backend-core/src/cloud/accounts.js

@@ -1,39 +0,0 @@
-const API = require("./api")
-const env = require("../environment")
-const { Headers } = require("../constants")
-
-const api = new API(env.ACCOUNT_PORTAL_URL)
-
-exports.getAccount = async email => {
-  const payload = {
-    email,
-  }
-  const response = await api.post(`/api/accounts/search`, {
-    body: payload,
-    headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
-    },
-  })
-  const json = await response.json()
-
-  if (response.status !== 200) {
-    throw new Error(`Error getting account by email ${email}`, json)
-  }
-
-  return json[0]
-}
-
-exports.getStatus = async () => {
-  const response = await api.get(`/api/status`, {
-    headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
-    },
-  })
-  const json = await response.json()
-
-  if (response.status !== 200) {
-    throw new Error(`Error getting status`)
-  }
-
-  return json
-}

packages/backend-core/src/cloud/accounts.ts

@@ -0,0 +1,63 @@
+import API from "./api"
+import env from "../environment"
+import { Headers } from "../constants"
+import { CloudAccount } from "@budibase/types"
+
+const api = new API(env.ACCOUNT_PORTAL_URL)
+
+export const getAccount = async (
+  email: string
+): Promise<CloudAccount | undefined> => {
+  const payload = {
+    email,
+  }
+  const response = await api.post(`/api/accounts/search`, {
+    body: payload,
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting account by email ${email}`)
+  }
+
+  const json: CloudAccount[] = await response.json()
+  return json[0]
+}
+
+export const getAccountByTenantId = async (
+  tenantId: string
+): Promise<CloudAccount | undefined> => {
+  const payload = {
+    tenantId,
+  }
+  const response = await api.post(`/api/accounts/search`, {
+    body: payload,
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting account by tenantId ${tenantId}`)
+  }
+
+  const json: CloudAccount[] = await response.json()
+  return json[0]
+}
+
+export const getStatus = async () => {
+  const response = await api.get(`/api/status`, {
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+  const json = await response.json()
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting status`)
+  }
+
+  return json
+}

packages/backend-core/src/events/identification.ts

@@ -7,12 +7,13 @@ import {
   Identity,
   IdentityType,
   Account,
-  AccountIdentity,
   BudibaseIdentity,
   isCloudAccount,
   isSSOAccount,
   TenantIdentity,
   SettingsConfig,
+  CloudAccount,
+  UserIdentity,
 } from "@budibase/types"
 import { analyticsProcessor } from "./processors"
 import * as dbUtils from "../db/utils"
@@ -81,20 +82,32 @@ const getHostingFromEnv = () => {
 
 export const identifyTenant = async (
   tenantId: string,
+  account: CloudAccount | undefined,
   timestamp?: string | number
 ) => {
   const global = await getGlobalIdentifiers(tenantId)
+  const id = global.id
+  const hosting = getHostingFromEnv()
+  const type = IdentityType.TENANT
+  const profession = account?.profession
+  const companySize = account?.size
 
   const identity: TenantIdentity = {
-    id: global.id,
+    id,
     tenantId: global.tenantId,
-    hosting: getHostingFromEnv(),
-    type: IdentityType.TENANT,
+    hosting,
+    type,
+    profession,
+    companySize,
   }
   await identify(identity, timestamp)
 }
 
-export const identifyUser = async (user: User, timestamp?: string | number) => {
+export const identifyUser = async (
+  user: User,
+  account: CloudAccount | undefined,
+  timestamp?: string | number
+) => {
   const id = user._id as string
   const tenantId = user.tenantId
   const hosting = env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
@@ -102,6 +115,10 @@ export const identifyUser = async (user: User, timestamp?: string | number) => {
   let builder = user.builder?.global
   let admin = user.admin?.global
   let providerType = user.providerType
+  const accountHolder = account?.budibaseUserId === user._id
+  const verified = account ? account.verified : false
+  const profession = account?.profession
+  const companySize = account?.size
 
   const identity: BudibaseIdentity = {
     id,
@@ -111,6 +128,10 @@ export const identifyUser = async (user: User, timestamp?: string | number) => {
     builder,
     admin,
     providerType,
+    accountHolder,
+    verified,
+    profession,
+    companySize,
   }
 
   await identify(identity, timestamp)
@@ -122,6 +143,10 @@ export const identifyAccount = async (account: Account) => {
   const hosting = account.hosting
   let type = IdentityType.USER
   let providerType = isSSOAccount(account) ? account.providerType : undefined
+  const verified = account.verified
+  const profession = account.profession
+  const companySize = account.size
+  const accountHolder = true
 
   if (isCloudAccount(account)) {
     if (account.budibaseUserId) {
@@ -130,15 +155,16 @@ export const identifyAccount = async (account: Account) => {
     }
   }
 
-  const identity: AccountIdentity = {
+  const identity: UserIdentity = {
     id,
     tenantId,
     hosting,
     type,
     providerType,
-    verified: account.verified,
-    profession: account.profession,
-    companySize: account.size,
+    verified,
+    profession,
+    companySize,
+    accountHolder,
   }
 
   await identify(identity)

packages/backend-core/src/events/processors/LoggingProcessor.ts

@@ -6,13 +6,19 @@ export default class LoggingProcessor implements EventProcessor {
   async processEvent(
     event: Event,
     identity: Identity,
-    properties: any
+    properties: any,
+    timestamp?: string
   ): Promise<void> {
     if (env.SELF_HOSTED && !env.isDev()) {
       return
     }
+    let timestampString = ""
+    if (timestamp) {
+      timestampString = `[timestamp=${new Date(timestamp).toISOString()}]`
+    }
+
     console.log(
-      `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${event}`
+      `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} `
     )
   }
 

packages/backend-core/src/events/publishers/account.ts

@@ -1,17 +1,29 @@
 import { publishEvent } from "../events"
-import { Event, Account } from "@budibase/types"
+import {
+  Event,
+  Account,
+  AccountCreatedEvent,
+  AccountDeletedEvent,
+  AccountVerifiedEvent,
+} from "@budibase/types"
 
 export async function created(account: Account) {
-  const properties = {}
+  const properties: AccountCreatedEvent = {
+    tenantId: account.tenantId,
+  }
   await publishEvent(Event.ACCOUNT_CREATED, properties)
 }
 
 export async function deleted(account: Account) {
-  const properties = {}
+  const properties: AccountDeletedEvent = {
+    tenantId: account.tenantId,
+  }
   await publishEvent(Event.ACCOUNT_DELETED, properties)
 }
 
 export async function verified(account: Account) {
-  const properties = {}
+  const properties: AccountVerifiedEvent = {
+    tenantId: account.tenantId,
+  }
   await publishEvent(Event.ACCOUNT_VERIFIED, properties)
 }

packages/backend-core/src/events/publishers/app.ts

@@ -16,58 +16,93 @@ import {
 } from "@budibase/types"
 
 export const created = async (app: App, timestamp?: string | number) => {
-  const properties: AppCreatedEvent = {}
+  const properties: AppCreatedEvent = {
+    appId: app.appId,
+    version: app.version,
+  }
   await publishEvent(Event.APP_CREATED, properties, timestamp)
 }
 
 export async function updated(app: App) {
-  const properties: AppUpdatedEvent = {}
+  const properties: AppUpdatedEvent = {
+    appId: app.appId,
+    version: app.version,
+  }
   await publishEvent(Event.APP_UPDATED, properties)
 }
 
 export async function deleted(app: App) {
-  const properties: AppDeletedEvent = {}
+  const properties: AppDeletedEvent = {
+    appId: app.appId,
+  }
   await publishEvent(Event.APP_DELETED, properties)
 }
 
 export async function published(app: App, timestamp?: string | number) {
-  const properties: AppPublishedEvent = {}
+  const properties: AppPublishedEvent = {
+    appId: app.appId,
+  }
   await publishEvent(Event.APP_PUBLISHED, properties, timestamp)
 }
 
 export async function unpublished(app: App) {
-  const properties: AppUnpublishedEvent = {}
+  const properties: AppUnpublishedEvent = {
+    appId: app.appId,
+  }
   await publishEvent(Event.APP_UNPUBLISHED, properties)
 }
 
 export async function fileImported(app: App) {
-  const properties: AppFileImportedEvent = {}
+  const properties: AppFileImportedEvent = {
+    appId: app.appId,
+  }
   await publishEvent(Event.APP_FILE_IMPORTED, properties)
 }
 
-export async function templateImported(templateKey: string) {
+export async function templateImported(app: App, templateKey: string) {
   const properties: AppTemplateImportedEvent = {
+    appId: app.appId,
     templateKey,
   }
   await publishEvent(Event.APP_TEMPLATE_IMPORTED, properties)
 }
 
-export async function versionUpdated(app: App) {
-  const properties: AppVersionUpdatedEvent = {}
+export async function versionUpdated(
+  app: App,
+  currentVersion: string,
+  updatedToVersion: string
+) {
+  const properties: AppVersionUpdatedEvent = {
+    appId: app.appId,
+    currentVersion,
+    updatedToVersion,
+  }
   await publishEvent(Event.APP_VERSION_UPDATED, properties)
 }
 
-export async function versionReverted(app: App) {
-  const properties: AppVersionRevertedEvent = {}
+export async function versionReverted(
+  app: App,
+  currentVersion: string,
+  revertedToVersion: string
+) {
+  const properties: AppVersionRevertedEvent = {
+    appId: app.appId,
+    currentVersion,
+    revertedToVersion,
+  }
   await publishEvent(Event.APP_VERSION_REVERTED, properties)
 }
 
 export async function reverted(app: App) {
-  const properties: AppRevertedEvent = {}
+  const properties: AppRevertedEvent = {
+    appId: app.appId,
+  }
   await publishEvent(Event.APP_REVERTED, properties)
 }
 
 export async function exported(app: App) {
-  const properties: AppExportedEvent = {}
+  const properties: AppExportedEvent = {
+    appId: app.appId,
+  }
   await publishEvent(Event.APP_EXPORTED, properties)
 }

packages/backend-core/src/index.ts

@@ -3,8 +3,8 @@ import errors from "./errors"
 import * as events from "./events"
 import * as migrations from "./migrations"
 import * as users from "./users"
+import * as accounts from "./cloud/accounts"
 import env from "./environment"
-import accounts from "./cloud/accounts"
 import tenancy from "./tenancy"
 import featureFlags from "./featureFlags"
 import sessions from "./security/sessions"

packages/server/src/api/controllers/application.ts

@@ -298,19 +298,19 @@ const creationEvents = async (request: any, app: App) => {
   const body = request.body
   if (body.useTemplate === "true") {
     // from template
-    if (body.templateKey) {
-      creationFns.push(app => events.app.templateImported(body.templateKey))
+    if (body.templateKey && body.templateKey !== "undefined") {
+      creationFns.push(a => events.app.templateImported(a, body.templateKey))
     }
     // from file
     else if (request.files?.templateFile) {
-      creationFns.push(events.app.fileImported)
+      creationFns.push(a => events.app.fileImported(a))
     }
     // unknown
     else {
       console.error("Could not determine template creation event")
     }
   }
-  creationFns.push(events.app.created)
+  creationFns.push(a => events.app.created(a))
 
   for (let fn of creationFns) {
     await fn(app)
@@ -381,12 +381,13 @@ export const updateClient = async (ctx: any) => {
   }
 
   // Update versions in app package
+  const updatedToVersion = packageJson.version
   const appPackageUpdates = {
-    version: packageJson.version,
+    version: updatedToVersion,
     revertableVersion: currentVersion,
   }
   const app = await updateAppPackage(appPackageUpdates, ctx.params.appId)
-  await events.app.versionUpdated(app)
+  await events.app.versionUpdated(app, currentVersion, updatedToVersion)
   ctx.status = 200
   ctx.body = app
 }
@@ -405,12 +406,14 @@ export const revertClient = async (ctx: any) => {
   }
 
   // Update versions in app package
+  const currentVersion = application.version
+  const revertedToVersion = application.revertableVersion
   const appPackageUpdates = {
-    version: application.revertableVersion,
+    version: revertedToVersion,
     revertableVersion: null,
   }
   const app = await updateAppPackage(appPackageUpdates, ctx.params.appId)
-  await events.app.versionReverted(app)
+  await events.app.versionReverted(app, currentVersion, revertedToVersion)
   ctx.status = 200
   ctx.body = app
 }

packages/server/src/api/controllers/backup.js

@@ -1,5 +1,6 @@
 const { streamBackup } = require("../../utilities/fileSystem")
-const { events } = require("@budibase/backend-core")
+const { events, context } = require("@budibase/backend-core")
+const { DocumentTypes } = require("../../db/utils")
 
 exports.exportAppDump = async function (ctx) {
   const { appId } = ctx.query
@@ -7,5 +8,10 @@ exports.exportAppDump = async function (ctx) {
   const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
   ctx.attachment(backupIdentifier)
   ctx.body = await streamBackup(appId)
-  await events.app.exported()
+
+  await context.doInAppContext(appId, async () => {
+    const appDb = context.getAppDB()
+    const app = await appDb.get(DocumentTypes.APP_METADATA)
+    await events.app.exported(app)
+  })
 }

packages/server/src/migrations/functions/backfill/global.ts

@@ -1,7 +1,9 @@
 import * as users from "./global/users"
 import * as rows from "./global/rows"
 import * as configs from "./global/configs"
-import { tenancy, events, migrations } from "@budibase/backend-core"
+import { tenancy, events, migrations, accounts } from "@budibase/backend-core"
+import { CloudAccount } from "@budibase/types"
+import env from "../../../environment"
 
 /**
  * Date:
@@ -14,12 +16,20 @@ import { tenancy, events, migrations } from "@budibase/backend-core"
 export const run = async (db: any) => {
   const tenantId = tenancy.getTenantId()
   const installTimestamp = (await getInstallTimestamp(db)) as number
+  let account: CloudAccount | undefined
+  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+    account = await accounts.getAccountByTenantId(tenantId)
+  }
 
-  await events.identification.identifyTenant(tenantId, installTimestamp)
+  await events.identification.identifyTenant(
+    tenantId,
+    account,
+    installTimestamp
+  )
   await configs.backfill(db, installTimestamp)
 
   // users and rows provide their own timestamp
-  await users.backfill(db)
+  await users.backfill(db, account)
   await rows.backfill()
 }
 

packages/server/src/migrations/functions/backfill/global/users.ts

@@ -1,5 +1,5 @@
 import { events, db as dbUtils } from "@budibase/backend-core"
-import { User } from "@budibase/types"
+import { User, CloudAccount } from "@budibase/types"
 
 // manually define user doc params - normally server doesn't read users from the db
 const getUserParams = (props: any) => {
@@ -15,12 +15,15 @@ export const getUsers = async (globalDb: any): Promise<User[]> => {
   return response.rows.map((row: any) => row.doc)
 }
 
-export const backfill = async (globalDb: any) => {
+export const backfill = async (
+  globalDb: any,
+  account: CloudAccount | undefined
+) => {
   const users = await getUsers(globalDb)
 
   for (const user of users) {
     const timestamp = user.createdAt as number
-    await events.identification.identifyUser(user, timestamp)
+    await events.identification.identifyUser(user, account, timestamp)
     await events.user.created(user, timestamp)
 
     if (user.admin?.global) {

packages/types/src/documents/app/app.ts

@@ -11,6 +11,7 @@ export interface App extends Document {
   instance: AppInstance
   tenantId: string
   status: string
+  revertableVersion?: string
 }
 
 export interface AppInstance {

packages/types/src/events/account.ts

@@ -0,0 +1,13 @@
+export interface AccountCreatedEvent {
+  tenantId: string
+  registrationStep?: string
+}
+
+export interface AccountDeletedEvent {
+  tenantId: string
+  registrationStep?: string
+}
+
+export interface AccountVerifiedEvent {
+  tenantId: string
+}

packages/types/src/events/app.ts

@@ -1,21 +1,50 @@
-export interface AppCreatedEvent {}
-
-export interface AppUpdatedEvent {}
-
-export interface AppDeletedEvent {}
-
-export interface AppPublishedEvent {}
-
-export interface AppUnpublishedEvent {}
-
-export interface AppFileImportedEvent {}
-
-export interface AppTemplateImportedEvent {}
-
-export interface AppVersionUpdatedEvent {}
-
-export interface AppVersionRevertedEvent {}
-
-export interface AppRevertedEvent {}
-
-export interface AppExportedEvent {}
+export interface AppCreatedEvent {
+  appId: string
+  version: string
+}
+
+export interface AppUpdatedEvent {
+  appId: string
+  version: string
+}
+
+export interface AppDeletedEvent {
+  appId: string
+}
+
+export interface AppPublishedEvent {
+  appId: string
+}
+
+export interface AppUnpublishedEvent {
+  appId: string
+}
+
+export interface AppFileImportedEvent {
+  appId: string
+}
+
+export interface AppTemplateImportedEvent {
+  appId: string
+  templateKey: string
+}
+
+export interface AppVersionUpdatedEvent {
+  appId: string
+  currentVersion: string
+  updatedToVersion: string
+}
+
+export interface AppVersionRevertedEvent {
+  appId: string
+  currentVersion: string
+  revertedToVersion: string
+}
+
+export interface AppRevertedEvent {
+  appId: string
+}
+
+export interface AppExportedEvent {
+  appId: string
+}

packages/types/src/events/identification.ts

@@ -13,11 +13,15 @@ export interface Identity {
 
 export interface TenantIdentity extends Identity {
   hosting: Hosting
+  profession?: string
+  companySize?: string
 }
 
 export interface UserIdentity extends TenantIdentity {
   hosting: Hosting
   type: IdentityType
+  verified: boolean
+  accountHolder: boolean
   providerType?: string
 }
 
@@ -25,9 +29,3 @@ export interface BudibaseIdentity extends UserIdentity {
   builder?: boolean
   admin?: boolean
 }
-
-export interface AccountIdentity extends UserIdentity {
-  verified: boolean
-  profession: string | undefined
-  companySize: string | undefined
-}

packages/types/src/events/index.ts

@@ -16,3 +16,4 @@ export * from "./table"
 export * from "./user"
 export * from "./view"
 export * from "./identification"
+export * from "./account"

packages/worker/src/api/controllers/global/users.ts

@@ -2,14 +2,13 @@ import { EmailTemplatePurpose } from "../../../constants"
 import { checkInviteCode } from "../../../utilities/redis"
 import { sendEmail } from "../../../utilities/email"
 import { users } from "../../../sdk"
-import { User } from "@budibase/types"
-import { events } from "@budibase/backend-core"
-import { getGlobalDB } from "@budibase/backend-core/dist/src/context"
+import env from "../../../environment"
+import { User, CloudAccount } from "@budibase/types"
+import { events, accounts, tenancy } from "@budibase/backend-core"
 
 const {
   errors,
   users: usersCore,
-  tenancy,
   db: dbUtils,
 } = require("@budibase/backend-core")
 
@@ -66,7 +65,11 @@ export const adminUser = async (ctx: any) => {
     ctx.body = await tenancy.doInTenant(tenantId, async () => {
       return users.save(user, hashPassword, requirePassword)
     })
-    await events.identification.identifyTenant(tenantId)
+    let account: CloudAccount | undefined
+    if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+      account = await accounts.getAccountByTenantId(tenantId)
+    }
+    await events.identification.identifyTenant(tenantId, account)
   } catch (err: any) {
     ctx.throw(err.status || 400, err)
   }
@@ -141,7 +144,7 @@ export const inviteAccept = async (ctx: any) => {
         email,
         ...info,
       })
-      const db = getGlobalDB()
+      const db = tenancy.getGlobalDB()
       const user = await db.get(saved._id)
       await events.user.inviteAccepted(user)
       return saved

packages/worker/src/sdk/users/users.ts

@@ -2,11 +2,10 @@ import env from "../../environment"
 import { quotas } from "@budibase/pro"
 import * as apps from "../../utilities/appService"
 import * as eventHelpers from "./events"
-import { User } from "@budibase/types"
+import { User, CloudAccount } from "@budibase/types"
 
 const {
   tenancy,
-  accounts,
   utils,
   db: dbUtils,
   constants,
@@ -17,7 +16,7 @@ const {
   HTTPError,
 } = require("@budibase/backend-core")
 
-import { events } from "@budibase/backend-core"
+import { events, accounts } from "@budibase/backend-core"
 
 /**
  * Retrieves all users from the current tenancy.
@@ -133,7 +132,14 @@ export const save = async (
     user._rev = response.rev
 
     await eventHelpers.handleSaveEvents(user, dbUser)
-    await events.identification.identifyUser(user)
+
+    // identify
+    let tenantAccount: CloudAccount | undefined
+    if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+      tenantAccount = await accounts.getAccountByTenantId(tenantId)
+    }
+    await events.identification.identifyUser(user, tenantAccount)
+
     await tenancy.tryAddTenant(tenantId, _id, email)
     await cache.user.invalidateUser(response.id)
     // let server know to sync user