Failover protection with event cache

packages/backend-core/src/cache/generic.js

@@ -1,5 +1,4 @@
 const redis = require("../redis/authRedis")
-const env = require("../environment")
 const { getTenantId } = require("../context")
 
 exports.CacheKeys = {
@@ -7,6 +6,8 @@ exports.CacheKeys = {
   INSTALLATION: "installation",
   ANALYTICS_ENABLED: "analyticsEnabled",
   UNIQUE_TENANT_ID: "uniqueTenantId",
+  EVENTS: "events",
+  BACKFILL_METADATA: "backfillMetadata",
 }
 
 exports.TTL = {
@@ -20,10 +21,41 @@ function generateTenantKey(key) {
   return `${key}:${tenantId}`
 }
 
-exports.withCache = async (key, ttl, fetchFn, opts = { useTenancy: true }) => {
+exports.keys = async pattern => {
+  const client = await redis.getCacheClient()
+  return client.keys(pattern)
+}
+
+/**
+ * Read only from the cache.
+ */
+exports.get = async (key, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  const value = await client.get(key)
+  return value
+}
+
+/**
+ * Write to the cache.
+ */
+exports.store = async (key, value, ttl, opts = { useTenancy: true }) => {
   key = opts.useTenancy ? generateTenantKey(key) : key
   const client = await redis.getCacheClient()
-  const cachedValue = await client.get(key)
+  await client.store(key, value, ttl)
+}
+
+exports.delete = async (key, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  return client.delete(key)
+}
+
+/**
+ * Read from the cache. Write to the cache if not exists.
+ */
+exports.withCache = async (key, ttl, fetchFn, opts = { useTenancy: true }) => {
+  const cachedValue = await exports.get(key, opts)
   if (cachedValue) {
     return cachedValue
   }
@@ -31,9 +63,7 @@ exports.withCache = async (key, ttl, fetchFn, opts = { useTenancy: true }) => {
   try {
     const fetchedValue = await fetchFn()
 
-    if (!env.isTest()) {
+    await exports.store(key, fetchedValue, ttl, opts)
-      await client.store(key, fetchedValue, ttl)
-    }
     return fetchedValue
   } catch (err) {
     console.error("Error fetching before cache - ", err)

packages/backend-core/src/db/tests/index.spec.js

@@ -1,5 +1,5 @@
 require("../../tests/utilities/TestConfiguration")
-const { dangerousGetDB, allDbs } = require("../")
+const { dangerousGetDB } = require("../")
 
 describe("db", () => { 
   

packages/backend-core/src/events/backfill.ts

@@ -0,0 +1,183 @@
+import { UserPermissionAssignedEvent } from "./../../../types/src/events/user"
+import {
+  Event,
+  BackfillMetadata,
+  CachedEvent,
+  SSOCreatedEvent,
+  AutomationCreatedEvent,
+  AutomationStepCreatedEvent,
+  DatasourceCreatedEvent,
+  LayoutCreatedEvent,
+  QueryCreatedEvent,
+  RoleCreatedEvent,
+  ScreenCreatedEvent,
+  TableCreatedEvent,
+  ViewCreatedEvent,
+  ViewCalculationCreatedEvent,
+  ViewFilterCreatedEvent,
+  AppPublishedEvent,
+  UserCreatedEvent,
+  RoleAssignedEvent,
+  RowsCreatedEvent,
+} from "@budibase/types"
+import * as context from "../context"
+import { CacheKeys } from "../cache/generic"
+import * as cache from "../cache/generic"
+
+// LIFECYCLE
+
+export const start = async (events: Event[]) => {
+  const metadata: BackfillMetadata = {
+    eventWhitelist: events,
+  }
+  return saveBackfillMetadata(metadata)
+}
+
+export const recordEvent = async (event: Event, properties: any) => {
+  const eventKey = getEventKey(event, properties)
+  // don't use a ttl - cleaned up by migration
+  // don't use tenancy - already in the key
+  await cache.store(eventKey, properties, undefined, { useTenancy: false })
+}
+
+export const end = async () => {
+  await deleteBackfillMetadata()
+  await clearEvents()
+}
+
+// CRUD
+
+const getBackfillMetadata = async (): Promise<BackfillMetadata | null> => {
+  return cache.get(CacheKeys.BACKFILL_METADATA)
+}
+
+const saveBackfillMetadata = async (
+  backfill: BackfillMetadata
+): Promise<void> => {
+  // no TTL - deleted by backfill
+  return cache.store(CacheKeys.BACKFILL_METADATA, backfill)
+}
+
+const deleteBackfillMetadata = async (): Promise<void> => {
+  await cache.delete(CacheKeys.BACKFILL_METADATA)
+}
+
+const clearEvents = async () => {
+  // wildcard
+  const pattern = getEventKey()
+  const keys = await cache.keys(pattern)
+
+  for (const key of keys) {
+    // delete each key
+    // don't  use tenancy, already in the key
+    await cache.delete(key, { useTenancy: false })
+  }
+}
+
+// HELPERS
+
+export const isBackfillingEvent = async (event: Event) => {
+  const backfill = await getBackfillMetadata()
+  const events = backfill?.eventWhitelist
+  if (events && events.includes(event)) {
+    return true
+  } else {
+    return false
+  }
+}
+
+export const isAlreadySent = async (event: Event, properties: any) => {
+  const eventKey = getEventKey(event, properties)
+  const cachedEvent: CachedEvent = await cache.get(eventKey, {
+    useTenancy: false,
+  })
+  return !!cachedEvent
+}
+
+const CUSTOM_PROPERTY_SUFFIX: any = {
+  // APP EVENTS
+  [Event.AUTOMATION_CREATED]: (properties: AutomationCreatedEvent) => {
+    return properties.automationId
+  },
+  [Event.AUTOMATION_STEP_CREATED]: (properties: AutomationStepCreatedEvent) => {
+    return properties.stepId
+  },
+  [Event.DATASOURCE_CREATED]: (properties: DatasourceCreatedEvent) => {
+    return properties.datasourceId
+  },
+  [Event.LAYOUT_CREATED]: (properties: LayoutCreatedEvent) => {
+    return properties.layoutId
+  },
+  [Event.QUERY_CREATED]: (properties: QueryCreatedEvent) => {
+    return properties.queryId
+  },
+  [Event.ROLE_CREATED]: (properties: RoleCreatedEvent) => {
+    return properties.roleId
+  },
+  [Event.SCREEN_CREATED]: (properties: ScreenCreatedEvent) => {
+    return properties.screenId
+  },
+  [Event.TABLE_CREATED]: (properties: TableCreatedEvent) => {
+    return properties.tableId
+  },
+  [Event.VIEW_CREATED]: (properties: ViewCreatedEvent) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.VIEW_CALCULATION_CREATED]: (
+    properties: ViewCalculationCreatedEvent
+  ) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.VIEW_FILTER_CREATED]: (properties: ViewFilterCreatedEvent) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.APP_PUBLISHED]: (properties: AppPublishedEvent) => {
+    return properties.appId // best uniqueness
+  },
+  [Event.APP_PUBLISHED]: (properties: AppPublishedEvent) => {
+    return properties.appId // best uniqueness
+  },
+  // GLOBAL EVENTS
+  [Event.AUTH_SSO_CREATED]: (properties: SSOCreatedEvent) => {
+    return properties.type
+  },
+  [Event.AUTH_SSO_ACTIVATED]: (properties: SSOCreatedEvent) => {
+    return properties.type
+  },
+  [Event.USER_CREATED]: (properties: UserCreatedEvent) => {
+    return properties.userId
+  },
+  [Event.USER_PERMISSION_ADMIN_ASSIGNED]: (
+    properties: UserPermissionAssignedEvent
+  ) => {
+    return properties.userId
+  },
+  [Event.USER_PERMISSION_BUILDER_ASSIGNED]: (
+    properties: UserPermissionAssignedEvent
+  ) => {
+    return properties.userId
+  },
+  [Event.ROLE_ASSIGNED]: (properties: RoleAssignedEvent) => {
+    return `${properties.roleId}-${properties.userId}`
+  },
+}
+
+const getEventKey = (event?: Event, properties?: any) => {
+  let eventKey: string
+
+  const tenantId = context.getTenantId()
+  if (event) {
+    eventKey = `${CacheKeys.EVENTS}:${tenantId}:${event}`
+
+    // use some properties to make the key more unique
+    const custom = CUSTOM_PROPERTY_SUFFIX[event]
+    const suffix = custom ? custom(properties) : undefined
+    if (suffix) {
+      eventKey = `${event}:${suffix}`
+    }
+  } else {
+    eventKey = `${CacheKeys.EVENTS}:${tenantId}:*`
+  }
+
+  return eventKey
+}

packages/backend-core/src/events/events.ts

@@ -1,6 +1,7 @@
 import { Event } from "@budibase/types"
 import { processors } from "./processors"
 import * as identification from "./identification"
+import * as backfill from "./backfill"
 
 export const publishEvent = async (
   event: Event,
@@ -9,5 +10,22 @@ export const publishEvent = async (
 ) => {
   // in future this should use async events via a distributed queue.
   const identity = await identification.getCurrentIdentity()
-  await processors.processEvent(event, identity, properties, timestamp)
+
+  const backfilling = await backfill.isBackfillingEvent(event)
+  // no backfill - send the event and exit
+  if (!backfilling) {
+    await processors.processEvent(event, identity, properties, timestamp)
+    return
+  }
+
+  // backfill active - check if the event has been sent already
+  const alreadySent = await backfill.isAlreadySent(event, properties)
+  if (alreadySent) {
+    // do nothing
+    return
+  } else {
+    // send and record the event
+    await processors.processEvent(event, identity, properties, timestamp)
+    await backfill.recordEvent(event, properties)
+  }
 }

packages/backend-core/src/events/index.ts

@@ -1,7 +1,10 @@
-import { processors } from "./processors"
 export * from "./publishers"
+export * as processors from "./processors"
 export * as analytics from "./analytics"
 export * as identification from "./identification"
+export * as backfillCache from "./backfill"
+
+import { processors } from "./processors"
 
 export const shutdown = () => {
   processors.shutdown()

packages/backend-core/src/events/processors/PosthogProcessor.ts

@@ -1,7 +1,8 @@
 import PostHog from "posthog-node"
-import { Event, Identity, Group } from "@budibase/types"
+import { Event, Identity, Group, BaseEvent } from "@budibase/types"
 import { EventProcessor } from "./types"
 import env from "../../environment"
+import context from "../../context"
 const pkg = require("../../../package.json")
 
 export default class PosthogProcessor implements EventProcessor {
@@ -17,12 +18,19 @@ export default class PosthogProcessor implements EventProcessor {
   async processEvent(
     event: Event,
     identity: Identity,
-    properties: any,
+    properties: BaseEvent,
     timestamp?: string | number
   ): Promise<void> {
     properties.version = pkg.version
     properties.service = env.SERVICE
+
+    const appId = context.getAppId()
+    if (appId) {
+      properties.appId = appId
+    }
+
     const payload: any = { distinctId: identity.id, event, properties }
+
     if (timestamp) {
       payload.timestamp = new Date(timestamp)
     }
@@ -32,9 +40,11 @@ export default class PosthogProcessor implements EventProcessor {
       payload.groups = {}
       if (identity.installationId) {
         payload.groups.installation = identity.installationId
+        payload.properties.installationId = identity.installationId
       }
       if (identity.tenantId) {
         payload.groups.tenant = identity.tenantId
+        payload.properties.tenantId = identity.tenantId
       }
     }
 

packages/backend-core/src/events/publishers/email.ts

@@ -1,20 +1,12 @@
 import { publishEvent } from "../events"
-import {
+import { Event, SMTPCreatedEvent, SMTPUpdatedEvent } from "@budibase/types"
-  Event,
-  SMTPConfig,
-  SMTPCreatedEvent,
-  SMTPUpdatedEvent,
-} from "@budibase/types"
 
-export async function SMTPCreated(
+export async function SMTPCreated(timestamp?: string | number) {
-  config: SMTPConfig,
-  timestamp?: string | number
-) {
   const properties: SMTPCreatedEvent = {}
   await publishEvent(Event.EMAIL_SMTP_CREATED, properties, timestamp)
 }
 
-export async function SMTPUpdated(config: SMTPConfig) {
+export async function SMTPUpdated() {
   const properties: SMTPUpdatedEvent = {}
   await publishEvent(Event.EMAIL_SMTP_UPDATED, properties)
 }

packages/backend-core/src/events/publishers/license.ts

@@ -4,7 +4,6 @@ import {
   License,
   LicenseActivatedEvent,
   LicenseDowngradedEvent,
-  LicenseQuotaExceededEvent,
   LicenseUpdatedEvent,
   LicenseUpgradedEvent,
 } from "@budibase/types"
@@ -32,12 +31,3 @@ export async function activated(license: License) {
   const properties: LicenseActivatedEvent = {}
   await publishEvent(Event.LICENSE_ACTIVATED, properties)
 }
-
-// TODO
-export async function quotaExceeded(quotaName: string, value: number) {
-  const properties: LicenseQuotaExceededEvent = {
-    name: quotaName,
-    value,
-  }
-  await publishEvent(Event.LICENSE_QUOTA_EXCEEDED, properties)
-}

packages/backend-core/src/events/publishers/query.ts

@@ -18,17 +18,32 @@ export const created = async (
   query: Query,
   timestamp?: string
 ) => {
-  const properties: QueryCreatedEvent = {}
+  const properties: QueryCreatedEvent = {
+    queryId: query._id as string,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
   await publishEvent(Event.QUERY_CREATED, properties, timestamp)
 }
 
 export const updated = async (datasource: Datasource, query: Query) => {
-  const properties: QueryUpdatedEvent = {}
+  const properties: QueryUpdatedEvent = {
+    queryId: query._id as string,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
   await publishEvent(Event.QUERY_UPDATED, properties)
 }
 
 export const deleted = async (datasource: Datasource, query: Query) => {
-  const properties: QueryDeletedEvent = {}
+  const properties: QueryDeletedEvent = {
+    queryId: query._id as string,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
   await publishEvent(Event.QUERY_DELETED, properties)
 }
 
@@ -37,7 +52,12 @@ export const imported = async (
   importSource: any,
   count: any
 ) => {
-  const properties: QueryImportedEvent = {}
+  const properties: QueryImportedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    count,
+    importSource,
+  }
   await publishEvent(Event.QUERY_IMPORT, properties)
 }
 
@@ -48,7 +68,12 @@ export const run = async (count: number, timestamp?: string | number) => {
   await publishEvent(Event.QUERIES_RUN, properties, timestamp)
 }
 
-export const previewed = async (datasource: Datasource) => {
+export const previewed = async (datasource: Datasource, query: Query) => {
-  const properties: QueryPreviewedEvent = {}
+  const properties: QueryPreviewedEvent = {
+    queryId: query._id,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
   await publishEvent(Event.QUERY_PREVIEWED, properties)
 }

packages/backend-core/src/events/publishers/role.ts

@@ -10,29 +10,45 @@ import {
   User,
 } from "@budibase/types"
 
-/* eslint-disable */
-
 export async function created(role: Role, timestamp?: string) {
-  const properties: RoleCreatedEvent = {}
+  const properties: RoleCreatedEvent = {
+    roleId: role._id as string,
+    permissionId: role.permissionId,
+    inherits: role.inherits,
+  }
   await publishEvent(Event.ROLE_CREATED, properties, timestamp)
 }
 
 export async function updated(role: Role) {
-  const properties: RoleUpdatedEvent = {}
+  const properties: RoleUpdatedEvent = {
+    roleId: role._id as string,
+    permissionId: role.permissionId,
+    inherits: role.inherits,
+  }
   await publishEvent(Event.ROLE_UPDATED, properties)
 }
 
 export async function deleted(role: Role) {
-  const properties: RoleDeletedEvent = {}
+  const properties: RoleDeletedEvent = {
+    roleId: role._id as string,
+    permissionId: role.permissionId,
+    inherits: role.inherits,
+  }
   await publishEvent(Event.ROLE_DELETED, properties)
 }
 
-export async function assigned(user: User, role: string, timestamp?: number) {
+export async function assigned(user: User, roleId: string, timestamp?: number) {
-  const properties: RoleAssignedEvent = {}
+  const properties: RoleAssignedEvent = {
+    userId: user._id as string,
+    roleId,
+  }
   await publishEvent(Event.ROLE_ASSIGNED, properties, timestamp)
 }
 
-export async function unassigned(user: User, role: string) {
+export async function unassigned(user: User, roleId: string) {
-  const properties: RoleUnassignedEvent = {}
+  const properties: RoleUnassignedEvent = {
+    userId: user._id as string,
+    roleId,
+  }
   await publishEvent(Event.ROLE_UNASSIGNED, properties)
 }

packages/backend-core/src/events/publishers/rows.ts

@@ -21,6 +21,10 @@ export const imported = async (
   format: RowImportFormat,
   count: number
 ) => {
-  const properties: RowsImportedEvent = {}
+  const properties: RowsImportedEvent = {
+    tableId: table._id as string,
+    format,
+    count,
+  }
   await publishEvent(Event.ROWS_IMPORTED, properties)
 }

packages/backend-core/src/events/publishers/screen.ts

@@ -7,11 +7,19 @@ import {
 } from "@budibase/types"
 
 export async function created(screen: Screen, timestamp?: string) {
-  const properties: ScreenCreatedEvent = {}
+  const properties: ScreenCreatedEvent = {
+    layoutId: screen.layoutId,
+    screenId: screen._id as string,
+    roleId: screen.routing.roleId,
+  }
   await publishEvent(Event.SCREEN_CREATED, properties, timestamp)
 }
 
 export async function deleted(screen: Screen) {
-  const properties: ScreenDeletedEvent = {}
+  const properties: ScreenDeletedEvent = {
+    layoutId: screen.layoutId,
+    screenId: screen._id as string,
+    roleId: screen.routing.roleId,
+  }
   await publishEvent(Event.SCREEN_DELETED, properties)
 }

packages/backend-core/src/events/publishers/serve.ts

@@ -7,8 +7,6 @@ import {
   AppServedEvent,
 } from "@budibase/types"
 
-/* eslint-disable */
-
 export async function servedBuilder() {
   const properties: BuilderServedEvent = {}
   await publishEvent(Event.SERVED_BUILDER, properties)
@@ -16,7 +14,6 @@ export async function servedBuilder() {
 
 export async function servedApp(app: App) {
   const properties: AppServedEvent = {
-    appId: app.appId,
     appVersion: app.version,
   }
   await publishEvent(Event.SERVED_APP, properties)

packages/backend-core/src/events/publishers/table.ts

@@ -11,29 +11,39 @@ import {
   TableImportedEvent,
 } from "@budibase/types"
 
-/* eslint-disable */
-
 export async function created(table: Table, timestamp?: string) {
-  const properties: TableCreatedEvent = {}
+  const properties: TableCreatedEvent = {
+    tableId: table._id as string,
+  }
   await publishEvent(Event.TABLE_CREATED, properties, timestamp)
 }
 
 export async function updated(table: Table) {
-  const properties: TableUpdatedEvent = {}
+  const properties: TableUpdatedEvent = {
+    tableId: table._id as string,
+  }
   await publishEvent(Event.TABLE_UPDATED, properties)
 }
 
 export async function deleted(table: Table) {
-  const properties: TableDeletedEvent = {}
+  const properties: TableDeletedEvent = {
+    tableId: table._id as string,
+  }
   await publishEvent(Event.TABLE_DELETED, properties)
 }
 
 export async function exported(table: Table, format: TableExportFormat) {
-  const properties: TableExportedEvent = {}
+  const properties: TableExportedEvent = {
+    tableId: table._id as string,
+    format,
+  }
   await publishEvent(Event.TABLE_EXPORTED, properties)
 }
 
 export async function imported(table: Table, format: TableImportFormat) {
-  const properties: TableImportedEvent = {}
+  const properties: TableImportedEvent = {
+    tableId: table._id as string,
+    format,
+  }
   await publishEvent(Event.TABLE_IMPORTED, properties)
 }

packages/backend-core/src/events/publishers/user.ts

@@ -15,27 +15,33 @@ import {
   UserUpdatedEvent,
 } from "@budibase/types"
 
-/* eslint-disable */
-
 export async function created(user: User, timestamp?: number) {
-  const properties: UserCreatedEvent = {}
+  const properties: UserCreatedEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(Event.USER_CREATED, properties, timestamp)
 }
 
 export async function updated(user: User) {
-  const properties: UserUpdatedEvent = {}
+  const properties: UserUpdatedEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(Event.USER_UPDATED, properties)
 }
 
 export async function deleted(user: User) {
-  const properties: UserDeletedEvent = {}
+  const properties: UserDeletedEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(Event.USER_DELETED, properties)
 }
 
 // PERMISSIONS
 
 export async function permissionAdminAssigned(user: User, timestamp?: number) {
-  const properties: UserPermissionAssignedEvent = {}
+  const properties: UserPermissionAssignedEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(
     Event.USER_PERMISSION_ADMIN_ASSIGNED,
     properties,
@@ -44,7 +50,9 @@ export async function permissionAdminAssigned(user: User, timestamp?: number) {
 }
 
 export async function permissionAdminRemoved(user: User) {
-  const properties: UserPermissionRemovedEvent = {}
+  const properties: UserPermissionRemovedEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(Event.USER_PERMISSION_ADMIN_REMOVED, properties)
 }
 
@@ -52,7 +60,9 @@ export async function permissionBuilderAssigned(
   user: User,
   timestamp?: number
 ) {
-  const properties: UserPermissionAssignedEvent = {}
+  const properties: UserPermissionAssignedEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(
     Event.USER_PERMISSION_BUILDER_ASSIGNED,
     properties,
@@ -61,40 +71,52 @@ export async function permissionBuilderAssigned(
 }
 
 export async function permissionBuilderRemoved(user: User) {
-  const properties: UserPermissionRemovedEvent = {}
+  const properties: UserPermissionRemovedEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(Event.USER_PERMISSION_BUILDER_REMOVED, properties)
 }
 
 // INVITE
 
-export async function invited(userInfo: any) {
+export async function invited() {
   const properties: UserInvitedEvent = {}
   await publishEvent(Event.USER_INVITED, properties)
 }
 
 export async function inviteAccepted(user: User) {
-  const properties: UserInviteAcceptedEvent = {}
+  const properties: UserInviteAcceptedEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(Event.USER_INVITED_ACCEPTED, properties)
 }
 
 // PASSWORD
 
 export async function passwordForceReset(user: User) {
-  const properties: UserPasswordForceResetEvent = {}
+  const properties: UserPasswordForceResetEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(Event.USER_PASSWORD_FORCE_RESET, properties)
 }
 
 export async function passwordUpdated(user: User) {
-  const properties: UserPasswordUpdatedEvent = {}
+  const properties: UserPasswordUpdatedEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(Event.USER_PASSWORD_UPDATED, properties)
 }
 
 export async function passwordResetRequested(user: User) {
-  const properties: UserPasswordResetRequestedEvent = {}
+  const properties: UserPasswordResetRequestedEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(Event.USER_PASSWORD_RESET_REQUESTED, properties)
 }
 
 export async function passwordReset(user: User) {
-  const properties: UserPasswordResetEvent = {}
+  const properties: UserPasswordResetEvent = {
+    userId: user._id as string,
+  }
   await publishEvent(Event.USER_PASSWORD_RESET, properties)
 }

packages/backend-core/src/events/publishers/view.ts

@@ -20,54 +20,75 @@ import {
 /* eslint-disable */
 
 export async function created(view: View, timestamp?: string) {
-  const properties: ViewCreatedEvent = {}
+  const properties: ViewCreatedEvent = {
+    tableId: view.tableId,
+  }
   await publishEvent(Event.VIEW_CREATED, properties, timestamp)
 }
 
 export async function updated(view: View) {
-  const properties: ViewUpdatedEvent = {}
+  const properties: ViewUpdatedEvent = {
+    tableId: view.tableId,
+  }
   await publishEvent(Event.VIEW_UPDATED, properties)
 }
 
-export async function deleted() {
+export async function deleted(view: View) {
-  const properties: ViewDeletedEvent = {}
+  const properties: ViewDeletedEvent = {
+    tableId: view.tableId,
+  }
   await publishEvent(Event.VIEW_DELETED, properties)
 }
 
 export async function exported(table: Table, format: TableExportFormat) {
-  const properties: ViewExportedEvent = {}
+  const properties: ViewExportedEvent = {
+    tableId: table._id as string,
+    format,
+  }
   await publishEvent(Event.VIEW_EXPORTED, properties)
 }
 
-export async function filterCreated(timestamp?: string) {
+export async function filterCreated(view: View, timestamp?: string) {
-  const properties: ViewFilterCreatedEvent = {}
+  const properties: ViewFilterCreatedEvent = {
+    tableId: view.tableId,
+  }
   await publishEvent(Event.VIEW_FILTER_CREATED, properties, timestamp)
 }
 
-export async function filterUpdated() {
+export async function filterUpdated(view: View) {
-  const properties: ViewFilterUpdatedEvent = {}
+  const properties: ViewFilterUpdatedEvent = {
+    tableId: view.tableId,
+  }
   await publishEvent(Event.VIEW_FILTER_UPDATED, properties)
 }
 
-export async function filterDeleted() {
+export async function filterDeleted(view: View) {
-  const properties: ViewFilterDeletedEvent = {}
+  const properties: ViewFilterDeletedEvent = {
+    tableId: view.tableId,
+  }
   await publishEvent(Event.VIEW_FILTER_DELETED, properties)
 }
 
-export async function calculationCreated(
+export async function calculationCreated(view: View, timestamp?: string) {
-  calculation: ViewCalculation,
+  const properties: ViewCalculationCreatedEvent = {
-  timestamp?: string
+    tableId: view.tableId,
-) {
+    calculation: view.calculation as ViewCalculation,
-  const properties: ViewCalculationCreatedEvent = {}
+  }
   await publishEvent(Event.VIEW_CALCULATION_CREATED, properties, timestamp)
 }
 
-export async function calculationUpdated() {
+export async function calculationUpdated(view: View) {
-  const properties: ViewCalculationUpdatedEvent = {}
+  const properties: ViewCalculationUpdatedEvent = {
+    tableId: view.tableId,
+    calculation: view.calculation as ViewCalculation,
+  }
   await publishEvent(Event.VIEW_CALCULATION_UPDATED, properties)
 }
 
-export async function calculationDeleted() {
+export async function calculationDeleted(existingView: View) {
-  const properties: ViewCalculationDeletedEvent = {}
+  const properties: ViewCalculationDeletedEvent = {
+    tableId: existingView.tableId,
+    calculation: existingView.calculation as ViewCalculation,
+  }
   await publishEvent(Event.VIEW_CALCULATION_DELETED, properties)
 }

packages/backend-core/src/redis/index.js

@@ -164,6 +164,10 @@ class RedisWrapper {
     return promisifyStream(stream)
   }
 
+  async keys(pattern) {
+    return CLIENT.keys(pattern)
+  }
+
   async get(key) {
     const db = this._db
     let response = await CLIENT.get(addDbPrefix(db, key))

packages/backend-core/src/tests/utilities/mocks/events.js

@@ -1,8 +1,15 @@
+const processors = require("../../../events/processors")
+
+jest.spyOn(processors.analyticsProcessor, "processEvent")
+
 const events = require("../../../events")
 
 jest.spyOn(events.identification, "identifyTenantGroup")
 jest.spyOn(events.identification, "identifyUser")
 
+jest.spyOn(events.backfill, "appSucceeded")
+jest.spyOn(events.backfill, "tenantSucceeded")
+
 jest.spyOn(events.account, "created")
 jest.spyOn(events.account, "deleted")
 jest.spyOn(events.account, "verified")
@@ -102,5 +109,3 @@ jest.spyOn(events.view, "filterDeleted")
 jest.spyOn(events.view, "calculationCreated")
 jest.spyOn(events.view, "calculationUpdated")
 jest.spyOn(events.view, "calculationDeleted")
-
-module.exports = events

packages/backend-core/src/tests/utilities/mocks/index.js

@@ -1,7 +1,6 @@
-const events = require("./events")
+require("./events")
 const date = require("./date")
 
 module.exports = {
-  events,
   date,
 }

packages/backend-core/src/tests/utils.spec.js

@@ -1,13 +1,17 @@
+require("./utilities/TestConfiguration")
 const { structures } = require("./utilities")
 const utils = require("../utils")
 const events = require("../events")
+const { doInTenant, DEFAULT_TENANT_ID }= require("../context")
 
 describe("utils", () => {
   describe("platformLogout", () => {
     it("should call platform logout", async () => {
-      const ctx = structures.koa.newContext()
+      await doInTenant(DEFAULT_TENANT_ID, async () => {
-      await utils.platformLogout({ ctx, userId: "test" })
+        const ctx = structures.koa.newContext()
-      expect(events.auth.logout).toBeCalledTimes(1)
+        await utils.platformLogout({ ctx, userId: "test" })
+        expect(events.auth.logout).toBeCalledTimes(1)
+      })
     })
   })
 })

packages/server/src/api/controllers/analytics.ts

@@ -1,7 +1,8 @@
 import { events } from "@budibase/backend-core"
 
 export const isEnabled = async (ctx: any) => {
+  const enabled = await events.analytics.enabled()
   ctx.body = {
-    enabled: events.analytics.enabled(),
+    enabled,
   }
 }

packages/server/src/api/controllers/automation.js

@@ -101,13 +101,13 @@ const handleStepEvents = async (oldAutomation, automation) => {
   // new steps
   const newSteps = getNewSteps(oldAutomation, automation)
   for (let step of newSteps) {
-    await events.automation.stepCreated(step)
+    await events.automation.stepCreated(automation, step)
   }
 
   // old steps
   const deletedSteps = getDeletedSteps(oldAutomation, automation)
   for (let step of deletedSteps) {
-    await events.automation.stepDeleted(step)
+    await events.automation.stepDeleted(automation, step)
   }
 }
 
@@ -134,7 +134,7 @@ exports.update = async function (ctx) {
       : {}
   // trigger has been updated, remove the test inputs
   if (oldAutoTrigger && oldAutoTrigger.id !== newAutoTrigger.id) {
-    await events.automation.triggerUpdated()
+    await events.automation.triggerUpdated(automation)
     await deleteEntityMetadata(
       ctx.appId,
       MetadataTypes.AUTOMATION_TEST_INPUT,
@@ -180,7 +180,7 @@ exports.destroy = async function (ctx) {
   // delete metadata first
   await cleanupAutomationMetadata(automationId)
   ctx.body = await db.remove(automationId, ctx.params.rev)
-  await events.automation.deleted()
+  await events.automation.deleted(oldAutomation)
 }
 
 exports.getActionList = async function (ctx) {
@@ -248,5 +248,5 @@ exports.test = async function (ctx) {
   })
   await clearTestFlag(automation._id)
   ctx.body = response
-  await events.automation.tested()
+  await events.automation.tested(automation)
 }

packages/server/src/api/controllers/datasource.js

@@ -109,7 +109,7 @@ exports.update = async function (ctx) {
   }
 
   const response = await db.put(datasource)
-  await events.datasource.updated()
+  await events.datasource.updated(datasource)
   datasource._rev = response.rev
 
   // Drain connection pools when configuration is changed
@@ -164,11 +164,11 @@ exports.save = async function (ctx) {
 
 exports.destroy = async function (ctx) {
   const db = getAppDB()
+  const datasourceId = ctx.params.datasourceId
 
+  const datasource = await db.get(datasourceId)
   // Delete all queries for the datasource
-  const queries = await db.allDocs(
+  const queries = await db.allDocs(getQueryParams(datasourceId, null))
-    getQueryParams(ctx.params.datasourceId, null)
-  )
   await db.bulkDocs(
     queries.rows.map(row => ({
       _id: row.id,
@@ -178,8 +178,8 @@ exports.destroy = async function (ctx) {
   )
 
   // delete the datasource
-  await db.remove(ctx.params.datasourceId, ctx.params.revId)
+  await db.remove(datasourceId, ctx.params.revId)
-  await events.datasource.deleted()
+  await events.datasource.deleted(datasource)
 
   ctx.message = `Datasource deleted.`
   ctx.status = 200

packages/server/src/api/controllers/dev.js

@@ -118,7 +118,7 @@ exports.revert = async ctx => {
     ctx.body = {
       message: "Reverted changes successfully.",
     }
-    await events.app.reverted()
+    await events.app.reverted(appDoc)
   } catch (err) {
     ctx.throw(400, `Unable to revert. ${err}`)
   } finally {

packages/server/src/api/controllers/query/import/tests/index.spec.js

@@ -53,7 +53,7 @@ describe("Rest Importer", () => {
   }
 
   const runTest = async (test, assertions) => {
-    config.doInContext(config.appId, async () => {
+    await config.doInContext(config.appId, async () => {
       for (let [key, data] of Object.entries(datasets)) {
         await test(key, data, assertions)
       }

packages/server/src/api/controllers/query/index.ts

@@ -114,10 +114,10 @@ export async function preview(ctx: any) {
   const db = getAppDB()
 
   const datasource = await db.get(ctx.request.body.datasourceId)
+  const query = ctx.request.body
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
-  const { fields, parameters, queryVerb, transformer, queryId } =
+  const { fields, parameters, queryVerb, transformer, queryId } = query
-    ctx.request.body
 
   try {
     const runFn = () =>
@@ -132,7 +132,7 @@ export async function preview(ctx: any) {
       })
 
     const { rows, keys, info, extra } = await quotas.addQuery(runFn)
-    await events.query.previewed(datasource)
+    await events.query.previewed(datasource, query)
     ctx.body = {
       rows,
       schemaFields: [...new Set(keys)],

packages/server/src/api/controllers/view/index.js

@@ -52,11 +52,11 @@ const calculationEvents = async (existingView, newView) => {
   const newCalculation = newView && newView.calculation
 
   if (existingCalculation && !newCalculation) {
-    await events.view.calculationDeleted()
+    await events.view.calculationDeleted(existingView)
   }
 
   if (!existingCalculation && newCalculation) {
-    await events.view.calculationCreated()
+    await events.view.calculationCreated(newView)
   }
 
   if (
@@ -64,7 +64,7 @@ const calculationEvents = async (existingView, newView) => {
     newCalculation &&
     existingCalculation !== newCalculation
   ) {
-    await events.view.calculationUpdated()
+    await events.view.calculationUpdated(newView)
   }
 }
 
@@ -77,11 +77,11 @@ const filterEvents = async (existingView, newView) => {
   const hasNewFilters = !!(newView && newView.filters && newView.filters.length)
 
   if (hasExistingFilters && !hasNewFilters) {
-    await events.view.filterDeleted()
+    await events.view.filterDeleted(newView)
   }
 
   if (!hasExistingFilters && hasNewFilters) {
-    await events.view.filterCreated()
+    await events.view.filterCreated(newView)
   }
 
   if (
@@ -89,15 +89,15 @@ const filterEvents = async (existingView, newView) => {
     hasNewFilters &&
     !isEqual(existingView.filters, newView.filters)
   ) {
-    await events.view.filterUpdated()
+    await events.view.filterUpdated(newView)
   }
 }
 
 const handleViewEvents = async (existingView, newView) => {
   if (!existingView) {
-    await events.view.created()
+    await events.view.created(newView)
   } else {
-    await events.view.updated()
+    await events.view.updated(newView)
   }
   await calculationEvents(existingView, newView)
   await filterEvents(existingView, newView)
@@ -110,7 +110,7 @@ exports.destroy = async ctx => {
   const table = await db.get(view.meta.tableId)
   delete table.views[viewName]
   await db.put(table)
-  await events.view.deleted()
+  await events.view.deleted(view)
 
   ctx.body = view
 }

packages/server/src/api/routes/tests/query.spec.js

@@ -201,15 +201,16 @@ describe("/queries", () => {
 
   describe("preview", () => {
     it("should be able to preview the query", async () => {
+      const query = {
+        datasourceId: datasource._id,
+        parameters: {},
+        fields: {},
+        queryVerb: "read",
+        name: datasource.name,
+      }
       const res = await request
         .post(`/api/queries/preview`)
-        .send({
+        .send(query)
-          datasourceId: datasource._id,
-          parameters: {},
-          fields: {},
-          queryVerb: "read",
-          name: datasource.name,
-        })
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
@@ -218,7 +219,7 @@ describe("/queries", () => {
       expect(res.body.rows.length).toEqual(1)
       expect(events.query.previewed).toBeCalledTimes(1)
       datasource.config = { schema: "public" }
-      expect(events.query.previewed).toBeCalledWith(datasource)
+      expect(events.query.previewed).toBeCalledWith(datasource, query)
     })
 
     it("should apply authorization to endpoint", async () => {

packages/server/src/migrations/functions/backfill/app.ts

@@ -6,7 +6,7 @@ import * as roles from "./app/roles"
 import * as tables from "./app/tables"
 import * as screens from "./app/screens"
 import * as global from "./global"
-import { App, AppBackfillSucceededEvent } from "@budibase/types"
+import { App, AppBackfillSucceededEvent, Event } from "@budibase/types"
 import { db as dbUtils, events } from "@budibase/backend-core"
 import env from "../../../environment"
 
@@ -22,6 +22,22 @@ const handleError = (e: any, errors?: any) => {
   throw e
 }
 
+const EVENTS = [
+  Event.AUTOMATION_CREATED,
+  Event.AUTOMATION_STEP_CREATED,
+  Event.DATASOURCE_CREATED,
+  Event.LAYOUT_CREATED,
+  Event.QUERY_CREATED,
+  Event.ROLE_CREATED,
+  Event.SCREEN_CREATED,
+  Event.TABLE_CREATED,
+  Event.VIEW_CREATED,
+  Event.VIEW_CALCULATION_CREATED,
+  Event.VIEW_FILTER_CREATED,
+  Event.APP_PUBLISHED,
+  Event.APP_CREATED,
+]
+
 /**
  * Date:
  * May 2022
@@ -39,6 +55,10 @@ export const run = async (appDb: any) => {
       return
     }
 
+    // tell the event pipeline to start caching
+    // events for this tenant
+    await events.backfillCache.start(EVENTS)
+
     const app: App = await appDb.get(dbUtils.DocumentTypes.APP_METADATA)
     const timestamp = app.createdAt as string
 
@@ -115,6 +135,8 @@ export const run = async (appDb: any) => {
     }
 
     await events.backfill.appSucceeded(properties)
+    // tell the event pipeline to stop caching events for this tenant
+    await events.backfillCache.end()
   } catch (e) {
     handleError(e)
     await events.backfill.appFailed(e)

packages/server/src/migrations/functions/backfill/app/layouts.ts

@@ -15,6 +15,13 @@ export const backfill = async (appDb: any, timestamp: string) => {
   const layouts: Layout[] = await getLayouts(appDb)
 
   for (const layout of layouts) {
+    // exclude default layouts
+    if (
+      layout._id === "layout_private_master" ||
+      layout._id === "layout_public_master"
+    ) {
+      continue
+    }
     await events.layout.created(layout, timestamp)
   }
 

packages/server/src/migrations/functions/backfill/app/tables.ts

@@ -22,11 +22,11 @@ export const backfill = async (appDb: any, timestamp: string) => {
         await events.view.created(view, timestamp)
 
         if (view.calculation) {
-          await events.view.calculationCreated(view.calculation, timestamp)
+          await events.view.calculationCreated(view, timestamp)
         }
 
         if (view.filters?.length) {
-          await events.view.filterCreated(timestamp)
+          await events.view.filterCreated(view, timestamp)
         }
       }
     }

packages/server/src/migrations/functions/backfill/global.ts

@@ -1,4 +1,3 @@
-import { TenantBackfillSucceededEvent } from "./../../../../../types/src/events/backfill"
 import * as users from "./global/users"
 import * as configs from "./global/configs"
 import * as quotas from "./global/quotas"
@@ -10,7 +9,12 @@ import {
   db as dbUtils,
 } from "@budibase/backend-core"
 import { QuotaUsage } from "@budibase/pro"
-import { CloudAccount, App } from "@budibase/types"
+import {
+  CloudAccount,
+  App,
+  TenantBackfillSucceededEvent,
+  Event,
+} from "@budibase/types"
 import env from "../../../environment"
 
 const failGraceful = env.SELF_HOSTED && !env.isDev()
@@ -57,6 +61,22 @@ const formatUsage = (usage: QuotaUsage) => {
   }
 }
 
+const EVENTS = [
+  Event.EMAIL_SMTP_CREATED,
+  Event.AUTH_SSO_CREATED,
+  Event.AUTH_SSO_ACTIVATED,
+  Event.ORG_NAME_UPDATED,
+  Event.ORG_LOGO_UPDATED,
+  Event.ORG_PLATFORM_URL_UPDATED,
+  Event.USER_CREATED,
+  Event.USER_PERMISSION_ADMIN_ASSIGNED,
+  Event.USER_PERMISSION_BUILDER_ASSIGNED,
+  Event.ROLE_ASSIGNED,
+  Event.ROWS_CREATED,
+  Event.QUERIES_RUN,
+  Event.AUTOMATIONS_RUN,
+]
+
 /**
  * Date:
  * May 2022
@@ -94,6 +114,10 @@ export const run = async (db: any) => {
       handleError(e, errors)
     }
 
+    // tell the event pipeline to start caching
+    // events for this tenant
+    await events.backfillCache.start(EVENTS)
+
     try {
       await configs.backfill(db, installTimestamp)
     } catch (e) {
@@ -130,7 +154,10 @@ export const run = async (db: any) => {
     } else {
       properties.errorCount = 0
     }
+
     await events.backfill.tenantSucceeded(properties)
+    // tell the event pipeline to stop caching events for this tenant
+    await events.backfillCache.end()
   } catch (e) {
     handleError(e)
     await events.backfill.tenantFailed(e)

packages/server/src/migrations/functions/backfill/global/configs.ts

@@ -28,7 +28,7 @@ export const backfill = async (
 
   for (const config of configs) {
     if (isSMTPConfig(config)) {
-      await events.email.SMTPCreated(config, timestamp)
+      await events.email.SMTPCreated(timestamp)
     }
     if (isGoogleConfig(config)) {
       await events.auth.SSOCreated("google", timestamp)

packages/server/src/migrations/tests/helpers.ts

@@ -26,5 +26,15 @@ export const saveSmtpConfig = async (globalDb: any) => {
 
 const saveConfig = async (config: Config, globalDb: any) => {
   config._id = db.generateConfigID({ type: config.type })
-  await globalDb.put(config)
+
+  let response
+  try {
+    response = await globalDb.get(config._id)
+    config._rev = response._rev
+    await globalDb.put(config)
+  } catch (e: any) {
+    if (e.status === 404) {
+      await globalDb.put(config)
+    }
+  }
 }

packages/server/src/migrations/tests/index.spec.ts

@@ -7,8 +7,6 @@ import * as helpers from "./helpers"
 const { mocks } = require("@budibase/backend-core/testUtils")
 const timestamp = mocks.date.MOCK_DATE.toISOString()
 
-jest.setTimeout(100000)
-
 describe("migrations", () => {
   const config = new TestConfig()
 
@@ -50,7 +48,7 @@ describe("migrations", () => {
         expect(events.automation.created).toBeCalledTimes(2)
         expect(events.automation.stepCreated).toBeCalledTimes(1)
         expect(events.datasource.created).toBeCalledTimes(2)
-        expect(events.layout.created).toBeCalledTimes(3)
+        expect(events.layout.created).toBeCalledTimes(1)
         expect(events.query.created).toBeCalledTimes(2)
         expect(events.role.created).toBeCalledTimes(2)
         expect(events.table.created).toBeCalledTimes(3)
@@ -58,6 +56,15 @@ describe("migrations", () => {
         expect(events.view.calculationCreated).toBeCalledTimes(1)
         expect(events.view.filterCreated).toBeCalledTimes(1)
         expect(events.screen.created).toBeCalledTimes(2)
+        expect(events.backfill.appSucceeded).toBeCalledTimes(2)
+
+        const processor = events.processors.analyticsProcessor.processEvent
+        console.log(processor)
+
+        // to make sure caching is working as expected
+        expect(
+          events.processors.analyticsProcessor.processEvent
+        ).toBeCalledTimes(23)
       })
     })
   })
@@ -96,6 +103,12 @@ describe("migrations", () => {
       expect(events.org.logoUpdated).toBeCalledTimes(1)
       expect(events.org.nameUpdated).toBeCalledTimes(1)
       expect(events.org.platformURLUpdated).toBeCalledTimes(1)
+      expect(events.backfill.tenantSucceeded).toBeCalledTimes(1)
+
+      // to make sure caching is working as expected
+      expect(events.processors.analyticsProcessor.processEvent).toBeCalledTimes(
+        19
+      )
     })
   })
 })

packages/server/src/tests/utilities/TestConfiguration.js

@@ -1,4 +1,3 @@
-require("./mocks")
 require("../../db").init()
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const env = require("../../environment")

packages/server/src/tests/utilities/mocks/core.js

@@ -1,3 +0,0 @@
-const utils = require("@budibase/backend-core/testUtils")
-const core = require("@budibase/backend-core")
-core.events = utils.mocks.events

packages/server/src/tests/utilities/mocks/index.js

@@ -1 +0,0 @@
-require("./core")

packages/types/src/documents/app/role.ts

@@ -1,3 +1,6 @@
 import { Document } from "../document"
 
-export interface Role extends Document {}
+export interface Role extends Document {
+  permissionId: string
+  inherits: string
+}

packages/types/src/documents/app/screen.ts

@@ -1,3 +1,9 @@
 import { Document } from "../document"
 
-export interface Screen extends Document {}
+export interface Screen extends Document {
+  layoutId: string
+  routing: {
+    route: string
+    roleId: string
+  }
+}

packages/types/src/events/account.ts

@@ -1,13 +1,15 @@
-export interface AccountCreatedEvent {
+import { BaseEvent } from "./event"
+
+export interface AccountCreatedEvent extends BaseEvent {
   tenantId: string
   registrationStep?: string
 }
 
-export interface AccountDeletedEvent {
+export interface AccountDeletedEvent extends BaseEvent {
   tenantId: string
   registrationStep?: string
 }
 
-export interface AccountVerifiedEvent {
+export interface AccountVerifiedEvent extends BaseEvent {
   tenantId: string
 }

packages/types/src/events/app.ts

@@ -1,50 +1,52 @@
-export interface AppCreatedEvent {
+import { BaseEvent } from "./event"
+
+export interface AppCreatedEvent extends BaseEvent {
   appId: string
   version: string
 }
 
-export interface AppUpdatedEvent {
+export interface AppUpdatedEvent extends BaseEvent {
   appId: string
   version: string
 }
 
-export interface AppDeletedEvent {
+export interface AppDeletedEvent extends BaseEvent {
   appId: string
 }
 
-export interface AppPublishedEvent {
+export interface AppPublishedEvent extends BaseEvent {
   appId: string
 }
 
-export interface AppUnpublishedEvent {
+export interface AppUnpublishedEvent extends BaseEvent {
   appId: string
 }
 
-export interface AppFileImportedEvent {
+export interface AppFileImportedEvent extends BaseEvent {
   appId: string
 }
 
-export interface AppTemplateImportedEvent {
+export interface AppTemplateImportedEvent extends BaseEvent {
   appId: string
   templateKey: string
 }
 
-export interface AppVersionUpdatedEvent {
+export interface AppVersionUpdatedEvent extends BaseEvent {
   appId: string
   currentVersion: string
   updatedToVersion: string
 }
 
-export interface AppVersionRevertedEvent {
+export interface AppVersionRevertedEvent extends BaseEvent {
   appId: string
   currentVersion: string
   revertedToVersion: string
 }
 
-export interface AppRevertedEvent {
+export interface AppRevertedEvent extends BaseEvent {
   appId: string
 }
 
-export interface AppExportedEvent {
+export interface AppExportedEvent extends BaseEvent {
   appId: string
 }

packages/types/src/events/auth.ts

@@ -1,27 +1,29 @@
+import { BaseEvent } from "./event"
+
 export type LoginSource = "local" | "google" | "oidc" | "google-internal"
 export type SSOType = "oidc" | "google"
 
-export interface LoginEvent {
+export interface LoginEvent extends BaseEvent {
   userId: string
   source: LoginSource
 }
 
-export interface LogoutEvent {
+export interface LogoutEvent extends BaseEvent {
   userId: string
 }
 
-export interface SSOCreatedEvent {
+export interface SSOCreatedEvent extends BaseEvent {
   type: SSOType
 }
 
-export interface SSOUpdatedEvent {
+export interface SSOUpdatedEvent extends BaseEvent {
   type: SSOType
 }
 
-export interface SSOActivatedEvent {
+export interface SSOActivatedEvent extends BaseEvent {
   type: SSOType
 }
 
-export interface SSODeactivatedEvent {
+export interface SSODeactivatedEvent extends BaseEvent {
   type: SSOType
 }

packages/types/src/events/automation.ts

@@ -1,32 +1,34 @@
-export interface AutomationCreatedEvent {
+import { BaseEvent } from "./event"
+
+export interface AutomationCreatedEvent extends BaseEvent {
   appId: string
   automationId: string
   triggerId: string
   triggerType: string
 }
 
-export interface AutomationTriggerUpdatedEvent {
+export interface AutomationTriggerUpdatedEvent extends BaseEvent {
   appId: string
   automationId: string
   triggerId: string
   triggerType: string
 }
 
-export interface AutomationDeletedEvent {
+export interface AutomationDeletedEvent extends BaseEvent {
   appId: string
   automationId: string
   triggerId: string
   triggerType: string
 }
 
-export interface AutomationTestedEvent {
+export interface AutomationTestedEvent extends BaseEvent {
   appId: string
   automationId: string
   triggerId: string
   triggerType: string
 }
 
-export interface AutomationStepCreatedEvent {
+export interface AutomationStepCreatedEvent extends BaseEvent {
   appId: string
   automationId: string
   triggerId: string
@@ -35,7 +37,7 @@ export interface AutomationStepCreatedEvent {
   stepType: string
 }
 
-export interface AutomationStepDeletedEvent {
+export interface AutomationStepDeletedEvent extends BaseEvent {
   appId: string
   automationId: string
   triggerId: string
@@ -44,6 +46,6 @@ export interface AutomationStepDeletedEvent {
   stepType: string
 }
 
-export interface AutomationsRunEvent {
+export interface AutomationsRunEvent extends BaseEvent {
   count: number
 }

packages/types/src/events/backfill.ts

@@ -1,4 +1,6 @@
-export interface AppBackfillSucceededEvent {
+import { BaseEvent, Event } from "./event"
+
+export interface AppBackfillSucceededEvent extends BaseEvent {
   appId: string
   automations: number
   datasources: number
@@ -11,11 +13,11 @@ export interface AppBackfillSucceededEvent {
   errorCount?: number
 }
 
-export interface AppBackfillFailedEvent {
+export interface AppBackfillFailedEvent extends BaseEvent {
   error: string
 }
 
-export interface TenantBackfillSucceededEvent {
+export interface TenantBackfillSucceededEvent extends BaseEvent {
   apps: number
   users: number
 
@@ -24,12 +26,21 @@ export interface TenantBackfillSucceededEvent {
   errorCount?: number
 }
 
-export interface TenantBackfillFailedEvent {
+export interface TenantBackfillFailedEvent extends BaseEvent {
   error: string
 }
 
-export interface InstallationBackfillSucceededEvent {}
+export interface InstallationBackfillSucceededEvent extends BaseEvent {}
 
-export interface InstallationBackfillFailedEvent {
+export interface InstallationBackfillFailedEvent extends BaseEvent {
   error: string
 }
+
+export interface BackfillMetadata extends BaseEvent {
+  eventWhitelist: Event[]
+}
+
+export interface CachedEvent extends BaseEvent {
+  event: Event
+  properties: any
+}

packages/types/src/events/config.ts

@@ -1,3 +0,0 @@
-export interface SMTPCreatedEvent {}
-
-export interface SMTPUpdatedEvent {}

packages/types/src/events/datasource.ts

@@ -1,14 +1,16 @@
-export interface DatasourceCreatedEvent {
+import { BaseEvent } from "./event"
+
+export interface DatasourceCreatedEvent extends BaseEvent {
   datasourceId: string
   source: string
 }
 
-export interface DatasourceUpdatedEvent {
+export interface DatasourceUpdatedEvent extends BaseEvent {
   datasourceId: string
   source: string
 }
 
-export interface DatasourceDeletedEvent {
+export interface DatasourceDeletedEvent extends BaseEvent {
   datasourceId: string
   source: string
 }

packages/types/src/events/email.ts

@@ -0,0 +1,5 @@
+import { BaseEvent } from "./event"
+
+export interface SMTPCreatedEvent extends BaseEvent {}
+
+export interface SMTPUpdatedEvent extends BaseEvent {}

packages/types/src/events/event.ts

@@ -37,7 +37,7 @@ export enum Event {
   ORG_LOGO_UPDATED = "org:info:logo:updated",
   ORG_PLATFORM_URL_UPDATED = "org:platformurl:updated",
 
-  // ORG / UPDATE
+  // VERSIONS
   VERSION_CHECKED = "version:checked",
   VERSION_UPGRADED = "version:upgraded",
   VERSION_DOWNGRADED = "version:downgraded",
@@ -134,7 +134,6 @@ export enum Event {
   LICENSE_DOWNGRADED = "license:downgraded",
   LICENSE_UPDATED = "license:updated",
   LICENSE_ACTIVATED = "license:activated",
-  LICENSE_QUOTA_EXCEEDED = "license:quota:exceeded",
 
   // ACCOUNT
   ACCOUNT_CREATED = "account:created",
@@ -150,6 +149,15 @@ export enum Event {
   INSTALLATION_BACKFILL_FAILED = "installation:backfill:failed",
 }
 
+// properties added at the final stage of the event pipeline
+export interface BaseEvent {
+  version?: string
+  service?: string
+  appId?: string
+  installationId?: string
+  tenantId?: string
+}
+
 export type RowImportFormat = "csv"
 export type TableExportFormat = "json" | "csv"
 export type TableImportFormat = "csv"

packages/types/src/events/index.ts

@@ -1,7 +1,7 @@
 export * from "./app"
 export * from "./auth"
 export * from "./automation"
-export * from "./config"
+export * from "./email"
 export * from "./datasource"
 export * from "./event"
 export * from "./layout"

packages/types/src/events/layout.ts

@@ -1,7 +1,9 @@
-export interface LayoutCreatedEvent {
+import { BaseEvent } from "./event"
+
+export interface LayoutCreatedEvent extends BaseEvent {
   layoutId: string
 }
 
-export interface LayoutDeletedEvent {
+export interface LayoutDeletedEvent extends BaseEvent {
   layoutId: string
 }

packages/types/src/events/license.ts

@@ -5,5 +5,3 @@ export interface LicenseDowngradedEvent {}
 export interface LicenseUpdatedEvent {}
 
 export interface LicenseActivatedEvent {}
-
-export interface LicenseQuotaExceededEvent {}

packages/types/src/events/query.ts

@@ -1,13 +1,40 @@
-export interface QueryCreatedEvent {}
+import { BaseEvent } from "./event"
 
-export interface QueryUpdatedEvent {}
+export interface QueryCreatedEvent extends BaseEvent {
+  queryId: string
+  datasourceId: string
+  source: string
+  queryVerb: string
+}
 
-export interface QueryDeletedEvent {}
+export interface QueryUpdatedEvent extends BaseEvent {
+  queryId: string
+  datasourceId: string
+  source: string
+  queryVerb: string
+}
 
-export interface QueryImportedEvent {}
+export interface QueryDeletedEvent extends BaseEvent {
+  queryId: string
+  datasourceId: string
+  source: string
+  queryVerb: string
+}
 
-export interface QueryPreviewedEvent {}
+export interface QueryImportedEvent extends BaseEvent {
+  datasourceId: string
+  source: string
+  count: number
+  importSource: string
+}
+
+export interface QueryPreviewedEvent extends BaseEvent {
+  queryId?: string
+  datasourceId: string
+  source: string
+  queryVerb: string
+}
 
-export interface QueriesRunEvent {
+export interface QueriesRunEvent extends BaseEvent {
   count: number
 }

packages/types/src/events/role.ts

@@ -1,9 +1,29 @@
-export interface RoleCreatedEvent {}
+import { BaseEvent } from "./event"
 
-export interface RoleUpdatedEvent {}
+export interface RoleCreatedEvent extends BaseEvent {
+  roleId: string
+  permissionId: string
+  inherits: string
+}
 
-export interface RoleDeletedEvent {}
+export interface RoleUpdatedEvent extends BaseEvent {
+  roleId: string
+  permissionId: string
+  inherits: string
+}
 
-export interface RoleAssignedEvent {}
+export interface RoleDeletedEvent extends BaseEvent {
+  roleId: string
+  permissionId: string
+  inherits: string
+}
 
-export interface RoleUnassignedEvent {}
+export interface RoleAssignedEvent extends BaseEvent {
+  userId: string
+  roleId: string
+}
+
+export interface RoleUnassignedEvent extends BaseEvent {
+  userId: string
+  roleId: string
+}

packages/types/src/events/rows.ts

@@ -1,5 +1,11 @@
-export interface RowsImportedEvent {}
+import { BaseEvent, RowImportFormat } from "./event"
 
-export interface RowsCreatedEvent {
+export interface RowsImportedEvent extends BaseEvent {
+  tableId: string
+  format: RowImportFormat
+  count: number
+}
+
+export interface RowsCreatedEvent extends BaseEvent {
   count: number
 }

packages/types/src/events/screen.ts

@@ -1,3 +1,13 @@
-export interface ScreenCreatedEvent {}
+import { BaseEvent } from "./event"
 
-export interface ScreenDeletedEvent {}
+export interface ScreenCreatedEvent extends BaseEvent {
+  screenId: string
+  layoutId: string
+  roleId: string
+}
+
+export interface ScreenDeletedEvent extends BaseEvent {
+  screenId: string
+  layoutId: string
+  roleId: string
+}

packages/types/src/events/serve.ts

@@ -1,11 +1,11 @@
-export interface BuilderServedEvent {}
+import { BaseEvent } from "./event"
 
-export interface AppServedEvent {
+export interface BuilderServedEvent extends BaseEvent {}
-  appId: string
+
+export interface AppServedEvent extends BaseEvent {
   appVersion: string
 }
 
-export interface AppPreviewServedEvent {
+export interface AppPreviewServedEvent extends BaseEvent {
-  appId: string
   appVersion: string
 }

packages/types/src/events/table.ts

@@ -1,9 +1,23 @@
-export interface TableCreatedEvent {}
+import { BaseEvent, TableExportFormat, TableImportFormat } from "./event"
 
-export interface TableUpdatedEvent {}
+export interface TableCreatedEvent extends BaseEvent {
+  tableId: string
+}
 
-export interface TableDeletedEvent {}
+export interface TableUpdatedEvent extends BaseEvent {
+  tableId: string
+}
 
-export interface TableExportedEvent {}
+export interface TableDeletedEvent extends BaseEvent {
+  tableId: string
+}
 
-export interface TableImportedEvent {}
+export interface TableExportedEvent extends BaseEvent {
+  tableId: string
+  format: TableExportFormat
+}
+
+export interface TableImportedEvent extends BaseEvent {
+  tableId: string
+  format: TableImportFormat
+}

packages/types/src/events/user.ts

@@ -1,21 +1,43 @@
-export interface UserCreatedEvent {}
+import { BaseEvent } from "./event"
 
-export interface UserUpdatedEvent {}
+export interface UserCreatedEvent extends BaseEvent {
+  userId: string
+}
 
-export interface UserDeletedEvent {}
+export interface UserUpdatedEvent extends BaseEvent {
+  userId: string
+}
 
-export interface UserPermissionAssignedEvent {}
+export interface UserDeletedEvent extends BaseEvent {
+  userId: string
+}
 
-export interface UserPermissionRemovedEvent {}
+export interface UserPermissionAssignedEvent extends BaseEvent {
+  userId: string
+}
 
-export interface UserInvitedEvent {}
+export interface UserPermissionRemovedEvent extends BaseEvent {
+  userId: string
+}
 
-export interface UserInviteAcceptedEvent {}
+export interface UserInvitedEvent extends BaseEvent {}
 
-export interface UserPasswordForceResetEvent {}
+export interface UserInviteAcceptedEvent extends BaseEvent {
+  userId: string
+}
 
-export interface UserPasswordUpdatedEvent {}
+export interface UserPasswordForceResetEvent extends BaseEvent {
+  userId: string
+}
 
-export interface UserPasswordResetRequestedEvent {}
+export interface UserPasswordUpdatedEvent extends BaseEvent {
+  userId: string
+}
 
-export interface UserPasswordResetEvent {}
+export interface UserPasswordResetRequestedEvent extends BaseEvent {
+  userId: string
+}
+
+export interface UserPasswordResetEvent extends BaseEvent {
+  userId: string
+}

packages/types/src/events/version.ts

@@ -1,8 +1,10 @@
-export interface VersionCheckedEvent {
+import { BaseEvent } from "./event"
+
+export interface VersionCheckedEvent extends BaseEvent {
   currentVersion: string
 }
 
-export interface VersionChangeEvent {
+export interface VersionChangeEvent extends BaseEvent {
   from: string
   to: string
 }

packages/types/src/events/view.ts

@@ -1,19 +1,46 @@
-export interface ViewCreatedEvent {}
+import { ViewCalculation } from "../documents"
-
+import { BaseEvent, TableExportFormat } from "./event"
-export interface ViewUpdatedEvent {}
+
-
+export interface ViewCreatedEvent extends BaseEvent {
-export interface ViewDeletedEvent {}
+  tableId: string
-
+}
-export interface ViewExportedEvent {}
+
-
+export interface ViewUpdatedEvent extends BaseEvent {
-export interface ViewFilterCreatedEvent {}
+  tableId: string
-
+}
-export interface ViewFilterUpdatedEvent {}
+
-
+export interface ViewDeletedEvent extends BaseEvent {
-export interface ViewFilterDeletedEvent {}
+  tableId: string
-
+}
-export interface ViewCalculationCreatedEvent {}
+
-
+export interface ViewExportedEvent extends BaseEvent {
-export interface ViewCalculationUpdatedEvent {}
+  tableId: string
-
+  format: TableExportFormat
-export interface ViewCalculationDeletedEvent {}
+}
+
+export interface ViewFilterCreatedEvent extends BaseEvent {
+  tableId: string
+}
+
+export interface ViewFilterUpdatedEvent extends BaseEvent {
+  tableId: string
+}
+
+export interface ViewFilterDeletedEvent extends BaseEvent {
+  tableId: string
+}
+
+export interface ViewCalculationCreatedEvent extends BaseEvent {
+  tableId: string
+  calculation: ViewCalculation
+}
+
+export interface ViewCalculationUpdatedEvent extends BaseEvent {
+  tableId: string
+  calculation: ViewCalculation
+}
+
+export interface ViewCalculationDeletedEvent extends BaseEvent {
+  tableId: string
+  calculation: ViewCalculation
+}

packages/worker/scripts/jestSetup.js

@@ -1,4 +1,11 @@
 const env = require("../src/environment")
+
+env._set("SELF_HOSTED", "1")
+env._set("NODE_ENV", "jest")
+env._set("JWT_SECRET", "test-jwtsecret")
+env._set("LOG_LEVEL", "silent")
+env._set("MULTI_TENANCY", true)
+
 const { mocks } = require("@budibase/backend-core/testUtils")
 
 // mock all dates to 2020-01-01T00:00:00.000Z
@@ -6,8 +13,4 @@ const { mocks } = require("@budibase/backend-core/testUtils")
 const tk = require("timekeeper")
 tk.freeze(mocks.date.MOCK_DATE)
 
-env._set("SELF_HOSTED", "1")
+global.console.log = jest.fn() // console.log are ignored in tests
-env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("LOG_LEVEL", "silent")
-env._set("MULTI_TENANCY", true)

packages/worker/src/api/controllers/global/users.ts

@@ -141,7 +141,7 @@ export const invite = async (ctx: any) => {
   ctx.body = {
     message: "Invitation has been sent.",
   }
-  await events.user.invited(userInfo)
+  await events.user.invited()
 }
 
 export const inviteAccept = async (ctx: any) => {

packages/worker/src/api/routes/tests/self.spec.js

@@ -1,6 +1,6 @@
 jest.mock("nodemailer")
 const { config, request } = require("../../../tests")
-const { events, utils } = require("@budibase/backend-core")
+const { events } = require("@budibase/backend-core")
 
 describe("/api/global/self", () => {
 

packages/worker/src/api/routes/tests/users.spec.js

@@ -39,7 +39,6 @@ describe("/api/global/users", () => {
     expect(sendMailMock).toHaveBeenCalled()
     expect(code).toBeDefined()
     expect(events.user.invited).toBeCalledTimes(1)
-    expect(events.user.invited).toBeCalledWith({ tenantId: structures.TENANT_ID })
   })
 
   it("should be able to create new user from invite", async () => {

packages/worker/src/tests/mocks/core.js

@@ -1,3 +0,0 @@
-const utils = require("@budibase/backend-core/testUtils")
-const core = require("@budibase/backend-core")
-core.events = utils.mocks.events

packages/worker/src/tests/mocks/index.js

@@ -1,4 +1,3 @@
-require("./core")
 const email = require("./email")
 
 module.exports = {