timestamp support, logging updates, migration progress indicators, identification updates

packages/backend-core/context.js

@@ -6,6 +6,7 @@ const {
   updateAppId,
   doInAppContext,
   doInUserContext,
+  doInTenant,
 } = require("./src/context")
 
 module.exports = {
@@ -16,4 +17,5 @@ module.exports = {
   updateAppId,
   doInAppContext,
   doInUserContext,
+  doInTenant,
 }

packages/backend-core/src/events/events.ts

@@ -2,8 +2,12 @@ import { Event } from "@budibase/types"
 import { processors } from "./processors"
 import * as identification from "./identification"
 
-export const publishEvent = async (event: Event, properties: any) => {
+export const publishEvent = async (
+  event: Event,
+  properties: any,
+  timestamp?: string | number
+) => {
   // in future this should use async events via a distributed queue.
   const identity = await identification.getCurrentIdentity()
-  await processors.processEvent(event, identity, properties)
+  await processors.processEvent(event, identity, properties, timestamp)
 }

packages/backend-core/src/events/identification.ts

@@ -23,18 +23,22 @@ export const getCurrentIdentity = async (): Promise<Identity> => {
   const user: SessionUser | undefined = context.getUser()
   let tenantId = context.getTenantId()
   let id: string
+  let type: IdentityType
 
   if (user) {
     id = user._id
+    type = IdentityType.USER
   } else {
     const global = await getGlobalIdentifiers(tenantId)
     id = global.id
     tenantId = global.tenantId
+    type = IdentityType.TENANT
   }
 
   return {
     id,
     tenantId,
+    type,
   }
 }
 
@@ -113,14 +117,13 @@ export const identifyAccount = async (account: Account) => {
   let id = account.accountId
   const tenantId = account.tenantId
   const hosting = account.hosting
-  let type = IdentityType.ACCOUNT
+  let type = IdentityType.USER
   let providerType = isSSOAccount(account) ? account.providerType : undefined
 
   if (isCloudAccount(account)) {
     if (account.budibaseUserId) {
       // use the budibase user as the id if set
       id = account.budibaseUserId
-      type = IdentityType.USER
     }
   }
 

packages/backend-core/src/events/processors/AnalyticsProcessor.ts

@@ -16,13 +16,14 @@ export default class AnalyticsProcessor implements EventProcessor {
   async processEvent(
     event: Event,
     identity: Identity,
-    properties: any
+    properties: any,
+    timestamp?: string
   ): Promise<void> {
     if (!(await analytics.enabled())) {
       return
     }
     if (this.posthog) {
-      this.posthog.processEvent(event, identity, properties)
+      this.posthog.processEvent(event, identity, properties, timestamp)
     }
   }
 

packages/backend-core/src/events/processors/LoggingProcessor.ts

@@ -1,5 +1,6 @@
 import { Event, Identity } from "@budibase/types"
 import { EventProcessor } from "./types"
+import env from "../../environment"
 
 export default class LoggingProcessor implements EventProcessor {
   async processEvent(
@@ -7,8 +8,11 @@ export default class LoggingProcessor implements EventProcessor {
     identity: Identity,
     properties: any
   ): Promise<void> {
+    if (env.SELF_HOSTED && !env.isDev()) {
+      return
+    }
     console.log(
-      `[audit] [tenant=${identity.tenantId}] [identity=${identity.id}] ${event}`
+      `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${event}`
     )
   }
 

packages/backend-core/src/events/processors/PosthogProcessor.ts

@@ -15,9 +15,14 @@ export default class PosthogProcessor implements EventProcessor {
   async processEvent(
     event: Event,
     identity: Identity,
-    properties: any
+    properties: any,
+    timestamp?: string | number
   ): Promise<void> {
-    this.posthog.capture({ distinctId: identity.id, event, properties })
+    const payload: any = { distinctId: identity.id, event, properties }
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+    this.posthog.capture(payload)
   }
 
   async identify(identity: Identity) {

packages/backend-core/src/events/processors/Processors.ts

@@ -12,10 +12,11 @@ export default class Processor implements EventProcessor {
   async processEvent(
     event: Event,
     identity: Identity,
-    properties: any
+    properties: any,
+    timestamp?: string | number
   ): Promise<void> {
     for (const eventProcessor of this.processors) {
-      await eventProcessor.processEvent(event, identity, properties)
+      await eventProcessor.processEvent(event, identity, properties, timestamp)
     }
   }
 

packages/backend-core/src/events/processors/types.ts

@@ -6,6 +6,11 @@ export enum EventProcessorType {
 }
 
 export interface EventProcessor {
-  processEvent(event: Event, identity: Identity, properties: any): Promise<void>
+  processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void>
   shutdown(): void
 }

packages/backend-core/src/migrations/index.js

@@ -31,6 +31,13 @@ exports.runMigration = async (migration, options = {}) => {
   const tenantId = getTenantId()
   const migrationType = migration.type
   const migrationName = migration.name
+  const silent = migration.silent
+
+  const log = message => {
+    if (!silent) {
+      console.log(message)
+    }
+  }
 
   // get the db to store the migration in
   let dbNames
@@ -45,8 +52,14 @@ exports.runMigration = async (migration, options = {}) => {
     )
   }
 
+  const length = dbNames.length
+  let count = 0
+
   // run the migration against each db
   for (const dbName of dbNames) {
+    count++
+    const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
+
     await doWithDB(dbName, async db => {
       try {
         const doc = await exports.getMigrationsDoc(db)
@@ -58,7 +71,7 @@ exports.runMigration = async (migration, options = {}) => {
             options.force[migrationType] &&
             options.force[migrationType].includes(migrationName)
           ) {
-            console.log(
+            log(
               `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
             )
           } else {
@@ -67,12 +80,12 @@ exports.runMigration = async (migration, options = {}) => {
           }
         }
 
-        console.log(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running`
+        log(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
         )
         // run the migration with tenant context
         await migration.fn(db)
-        console.log(
+        log(
           `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
         )
 
@@ -91,7 +104,6 @@ exports.runMigration = async (migration, options = {}) => {
 }
 
 exports.runMigrations = async (migrations, options = {}) => {
-  console.log("Running migrations")
   let tenantIds
   if (environment.MULTI_TENANCY) {
     if (!options.tenantIds || !options.tenantIds.length) {
@@ -105,8 +117,19 @@ exports.runMigrations = async (migrations, options = {}) => {
     tenantIds = [DEFAULT_TENANT_ID]
   }
 
+  if (tenantIds.length > 1) {
+    console.log(`Checking migrations for ${tenantIds.length} tenants`)
+  } else {
+    console.log("Checking migrations")
+  }
+
+  let count = 0
   // for all tenants
   for (const tenantId of tenantIds) {
+    count++
+    if (tenantIds.length > 1) {
+      console.log(`Progress [${count}/${tenantIds.length}]`)
+    }
     // for all migrations
     for (const migration of migrations) {
       // run the migration

packages/server/src/migrations/functions/backfill/app.ts

@@ -6,6 +6,7 @@ import * as queries from "./app/queries"
 import * as roles from "./app/roles"
 import * as tables from "./app/tables"
 import * as screens from "./app/screens"
+import * as global from "./global"
 
 /**
  * Date:
@@ -16,6 +17,13 @@ import * as screens from "./app/screens"
  */
 
 export const run = async (appDb: any) => {
+  if (await global.isComplete()) {
+    // make sure new apps aren't backfilled
+    // return if the global migration for this tenant is complete
+    // which runs after the app migrations
+    return
+  }
+
   await apps.backfill(appDb)
   await automations.backfill(appDb)
   await datasources.backfill(appDb)

packages/server/src/migrations/functions/backfill/global.ts

@@ -1,6 +1,7 @@
 import * as users from "./global/users"
 import * as rows from "./global/rows"
 import * as configs from "./global/configs"
+import { tenancy, events, migrations } from "@budibase/backend-core"
 
 /**
  * Date:
@@ -11,7 +12,16 @@ import * as configs from "./global/configs"
  */
 
 export const run = async (db: any) => {
+  const tenantId = tenancy.getTenantId()
+  await events.identification.identifyTenant(tenantId)
+
   await users.backfill(db)
   await rows.backfill()
   await configs.backfill(db)
 }
+
+export const isComplete = async (): Promise<boolean> => {
+  const globalDb = tenancy.getGlobalDB()
+  const migrationsDoc = await migrations.getMigrationsDoc(globalDb)
+  return !!migrationsDoc.event_global_backfill
+}

packages/server/src/migrations/functions/backfill/global/users.ts

@@ -19,6 +19,7 @@ export const backfill = async (globalDb: any) => {
   const users = await getUsers(globalDb)
 
   for (const user of users) {
+    await events.identification.identifyUser(user)
     await events.user.created(user)
 
     if (user.admin?.global) {

packages/server/src/migrations/index.ts

@@ -7,12 +7,14 @@ import * as appUrls from "./functions/appUrls"
 import * as developerQuota from "./functions/developerQuota"
 import * as publishedAppsQuota from "./functions/publishedAppsQuota"
 import * as backfill from "./functions/backfill"
+import env from "../environment"
 
 export interface Migration {
   type: string
   name: string
   opts?: object
   fn: Function
+  silent?: boolean
 }
 
 /**
@@ -58,16 +60,18 @@ export const MIGRATIONS: Migration[] = [
     name: "published_apps_quota",
     fn: publishedAppsQuota.run,
   },
-  {
-    type: migrations.MIGRATION_TYPES.GLOBAL,
-    name: "event_global_backfill",
-    fn: backfill.global.run,
-  },
   {
     type: migrations.MIGRATION_TYPES.APP,
     name: "event_app_backfill",
     opts: { all: true },
     fn: backfill.app.run,
+    silent: !!env.SELF_HOSTED, // reduce noisy logging
+  },
+  {
+    type: migrations.MIGRATION_TYPES.GLOBAL,
+    name: "event_global_backfill",
+    fn: backfill.global.run,
+    silent: !!env.SELF_HOSTED, // reduce noisy logging
   },
 ]
 

packages/types/src/events/identification.ts

@@ -2,18 +2,17 @@ import { Hosting } from "../core"
 
 export enum IdentityType {
   USER = "user", // cloud and self hosted users
-  ACCOUNT = "account", // self hosted accounts
   TENANT = "tenant", // cloud and self hosted tenants
 }
 
 export interface Identity {
   id: string
   tenantId: string
+  type: IdentityType
 }
 
 export interface TenantIdentity extends Identity {
   hosting: Hosting
-  type: IdentityType
 }
 
 export interface UserIdentity extends TenantIdentity {