Migration for apps, automations, datasources, layouts, queries, roles, tables

packages/backend-core/src/events/publishers/role.ts

@@ -27,12 +27,12 @@ export function deleted(role: Role) {
   publishEvent(Event.ROLE_DELETED, properties)
 }
 
-export function assigned(user: User, role: Role) {
+export function assigned(user: User, role: string) {
   const properties: RoleAssignedEvent = {}
   publishEvent(Event.ROLE_ASSIGNED, properties)
 }
 
-export function unassigned(user: User, role: Role) {
+export function unassigned(user: User, role: string) {
   const properties: RoleUnassignedEvent = {}
   publishEvent(Event.ROLE_UNASSIGNED, properties)
 }

packages/backend-core/src/migrations/index.js

@@ -27,7 +27,7 @@ exports.getMigrationsDoc = async db => {
   }
 }
 
-const runMigration = async (migration, options = {}) => {
+exports.runMigration = async (migration, options = {}) => {
   const tenantId = getTenantId()
   const migrationType = migration.type
   const migrationName = migration.name
@@ -110,7 +110,7 @@ exports.runMigrations = async (migrations, options = {}) => {
     // for all migrations
     for (const migration of migrations) {
       // run the migration
-      await doInTenant(tenantId, () => runMigration(migration, options))
+      await doInTenant(tenantId, () => exports.runMigration(migration, options))
     }
   }
   console.log("Migrations complete")

packages/server/src/migrations/functions/backfill/app.ts

@@ -1,4 +1,10 @@
-import * as app from "./app/app"
+import * as apps from "./app/apps"
+import * as automations from "./app/automations"
+import * as datasources from "./app/datasources"
+import * as layouts from "./app/layouts"
+import * as queries from "./app/queries"
+import * as roles from "./app/roles"
+import * as tables from "./app/tables"
 
 /**
  * Date:
@@ -9,5 +15,11 @@ import * as app from "./app/app"
  */
 
 export const run = async (appDb: any) => {
-  await app.backfill(appDb)
+  await apps.backfill(appDb)
+  await automations.backfill(appDb)
+  await datasources.backfill(appDb)
+  await layouts.backfill(appDb)
+  await queries.backfill(appDb)
+  await roles.backfill(appDb)
+  await tables.backfill(appDb)
 }

packages/server/src/migrations/functions/backfill/app/automation.ts

@@ -1,15 +0,0 @@
-import { events, db } from "@budibase/backend-core"
-import { Automation, AutomationStep } from "@budibase/types"
-
-export const backfill = async (appDb: any) => {
-  const automations: Automation[] = []
-
-  for (const automation of automations) {
-    events.automation.created(automation)
-
-    const steps: AutomationStep[] = []
-    for (const step of steps) {
-      events.automation.stepCreated(automation, step)
-    }
-  }
-}

packages/server/src/migrations/functions/backfill/app/automations.ts

@@ -0,0 +1,26 @@
+import { events, db } from "@budibase/backend-core"
+import { getAutomationParams } from "../../../../db/utils"
+import { Automation } from "@budibase/types"
+
+const getAutomations = async (appDb: any): Promise<Automation[]> => {
+  const response = await appDb.allDocs(
+    getAutomationParams(null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map((row: any) => row.doc)
+}
+
+export const backfill = async (appDb: any) => {
+  if (db.isDevAppID(appDb.name)) {
+    const automations = await getAutomations(appDb)
+
+    for (const automation of automations) {
+      events.automation.created(automation)
+
+      for (const step of automation.definition.steps) {
+        events.automation.stepCreated(automation, step)
+      }
+    }
+  }
+}

packages/server/src/migrations/functions/backfill/app/datasource.ts

@@ -1,10 +0,0 @@
-import { events, db } from "@budibase/backend-core"
-import { Datasource } from "@budibase/types"
-
-export const backfill = async (appDb: any) => {
-  const datasources: Datasource[] = []
-
-  for (const datasource of datasources) {
-    events.datasource.created(datasource)
-  }
-}

packages/server/src/migrations/functions/backfill/app/datasources.ts

@@ -0,0 +1,22 @@
+import { events, db } from "@budibase/backend-core"
+import { getDatasourceParams } from "../../../../db/utils"
+import { Datasource } from "@budibase/types"
+
+const getDatasources = async (appDb: any): Promise<Datasource[]> => {
+  const response = await appDb.allDocs(
+    getDatasourceParams(null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map((row: any) => row.doc)
+}
+
+export const backfill = async (appDb: any) => {
+  if (db.isDevAppID(appDb.name)) {
+    const datasources: Datasource[] = await getDatasources(appDb)
+
+    for (const datasource of datasources) {
+      events.datasource.created(datasource)
+    }
+  }
+}

packages/server/src/migrations/functions/backfill/app/layout.ts

@@ -1,10 +0,0 @@
-import { events, db } from "@budibase/backend-core"
-import { Layout } from "@budibase/types"
-
-export const backfill = async (appDb: any) => {
-  const layouts: Layout[] = []
-
-  for (const layout of layouts) {
-    events.layout.created(layout)
-  }
-}

packages/server/src/migrations/functions/backfill/app/layouts.ts

@@ -0,0 +1,22 @@
+import { events, db } from "@budibase/backend-core"
+import { getLayoutParams } from "../../../../db/utils"
+import { Layout } from "@budibase/types"
+
+const getLayouts = async (appDb: any): Promise<Layout[]> => {
+  const response = await appDb.allDocs(
+    getLayoutParams(null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map((row: any) => row.doc)
+}
+
+export const backfill = async (appDb: any) => {
+  if (db.isDevAppID(appDb.name)) {
+    const layouts: Layout[] = await getLayouts(appDb)
+
+    for (const layout of layouts) {
+      events.layout.created(layout)
+    }
+  }
+}

packages/server/src/migrations/functions/backfill/app/queries.ts

@@ -0,0 +1,33 @@
+import { events, db } from "@budibase/backend-core"
+import { getQueryParams } from "../../../../db/utils"
+import { Query, Datasource } from "@budibase/types"
+
+const getQueries = async (appDb: any): Promise<Query[]> => {
+  const response = await appDb.allDocs(
+    getQueryParams(null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map((row: any) => row.doc)
+}
+
+const getDatasource = async (
+  appDb: any,
+  datasourceId: string
+): Promise<Datasource> => {
+  return appDb.get(datasourceId)
+}
+
+export const backfill = async (appDb: any) => {
+  if (db.isDevAppID(appDb.name)) {
+    const queries: Query[] = await getQueries(appDb)
+
+    for (const query of queries) {
+      const datasource: Datasource = await getDatasource(
+        appDb,
+        query.datasourceId
+      )
+      events.query.created(datasource, query)
+    }
+  }
+}

packages/server/src/migrations/functions/backfill/app/query.ts

@@ -1,11 +0,0 @@
-import { events, db } from "@budibase/backend-core"
-import { Query, Datasource } from "@budibase/types"
-
-export const backfill = async (appDb: any) => {
-  const queries: Query[] = []
-
-  for (const query of queries) {
-    const datasource: Datasource = {}
-    events.query.created(datasource, query)
-  }
-}

packages/server/src/migrations/functions/backfill/app/role.ts

@@ -1,12 +0,0 @@
-import { events, db } from "@budibase/backend-core"
-import { Role, User } from "@budibase/types"
-
-export const backfill = async (appDb: any) => {
-  const roles: Role[] = []
-
-  for (const role of roles) {
-    events.role.created(role)
-    const user: User = {}
-    events.role.assigned(user, role)
-  }
-}

packages/server/src/migrations/functions/backfill/app/roles.ts

@@ -0,0 +1,22 @@
+import { events, db } from "@budibase/backend-core"
+import { getRoleParams } from "../../../../db/utils"
+import { Role } from "@budibase/types"
+
+const getRoles = async (appDb: any): Promise<Role[]> => {
+  const response = await appDb.allDocs(
+    getRoleParams(null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map((row: any) => row.doc)
+}
+
+export const backfill = async (appDb: any) => {
+  if (db.isDevAppID(appDb.name)) {
+    const roles = await getRoles(appDb)
+
+    for (const role of roles) {
+      events.role.created(role)
+    }
+  }
+}

packages/server/src/migrations/functions/backfill/app/table.ts

@@ -1,3 +0,0 @@
-// TABLE_CREATED = "table:created",
-// <!-- maybe -->
-// TABLE_DATA_IMPORTED = "table:data:imported",

packages/server/src/migrations/functions/backfill/app/tables.ts

@@ -0,0 +1,22 @@
+import { events, db } from "@budibase/backend-core"
+import { getTableParams } from "../../../../db/utils"
+import { Table } from "@budibase/types"
+
+const getTables = async (appDb: any): Promise<Table[]> => {
+  const response = await appDb.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map((row: any) => row.doc)
+}
+
+export const backfill = async (appDb: any) => {
+  if (db.isDevAppID(appDb.name)) {
+    const tables = await getTables(appDb)
+
+    for (const table of tables) {
+      events.table.created(table)
+    }
+  }
+}

packages/server/src/migrations/functions/backfill/tests/app.spec.ts

@@ -1,42 +0,0 @@
-import { tenancy, events } from "@budibase/backend-core"
-import TestConfig from "../../../../tests/utilities/TestConfiguration"
-import { backfill } from "../app/app"
-
-describe("app backfill", () => {
-  const config = new TestConfig()
-
-  beforeEach(async () => {
-    await config.init()
-    jest.clearAllMocks()
-  })
-
-  afterEach(() => {
-    config.end()
-  })
-
-  it("should backfill dev app", async () => {
-    await config.doInContext(null, async () => {
-      const db = tenancy.getAppDB({})
-
-      await backfill(db)
-
-      expect(events.app.created).toBeCalledTimes(1)
-      expect(events.app.published).toBeCalledTimes(0)
-    })
-  })
-
-  it("should backfill prod app", async () => {
-    await config.doInContext(
-      null,
-      async () => {
-        const db = tenancy.getAppDB({})
-
-        await backfill(db)
-
-        // expect(events.app.created).toBeCalledTimes(0)
-        expect(events.app.published).toBeCalledTimes(1)
-      },
-      { prod: true }
-    )
-  })
-})

packages/server/src/migrations/tests/index.spec.ts

@@ -0,0 +1,52 @@
+import { events, migrations } from "@budibase/backend-core"
+import TestConfig from "../../tests/utilities/TestConfiguration"
+import structures from "../../tests/utilities/structures"
+import { MIGRATIONS } from "../"
+
+jest.setTimeout(100000)
+
+describe("migrations", () => {
+  const config = new TestConfig()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll(() => {
+    config.end()
+  })
+
+  describe("backfill", () => {
+    it("runs app db migration", async () => {
+      await config.doInContext(null, async () => {
+        await config.createAutomation()
+        await config.createAutomation(structures.newAutomation())
+        await config.createDatasource()
+        await config.createDatasource()
+        await config.createLayout()
+        await config.createQuery()
+        await config.createQuery()
+        await config.createRole()
+        await config.createRole()
+        await config.createTable()
+        await config.createTable()
+
+        jest.clearAllMocks()
+        const migration = MIGRATIONS.filter(
+          m => m.name === "event_app_backfill"
+        )[0]
+        await migrations.runMigration(migration)
+
+        expect(events.app.created).toBeCalledTimes(1)
+        expect(events.app.published).toBeCalledTimes(1)
+        expect(events.automation.created).toBeCalledTimes(2)
+        expect(events.automation.stepCreated).toBeCalledTimes(1)
+        expect(events.datasource.created).toBeCalledTimes(2)
+        expect(events.layout.created).toBeCalledTimes(3)
+        expect(events.query.created).toBeCalledTimes(2)
+        expect(events.role.created).toBeCalledTimes(2)
+        expect(events.table.created).toBeCalledTimes(3)
+      })
+    })
+  })
+})

packages/types/src/documents/app/automation.ts

@@ -1,4 +1,9 @@
-export interface Automation {}
+export interface Automation {
+  definition: {
+    steps: AutomationStep[]
+    trigger: AutomationTrigger
+  }
+}
 
 export interface AutomationStep {}
 

packages/types/src/documents/app/index.ts

@@ -9,3 +9,4 @@ export * from "./screen"
 export * from "./view"
 export * from "./document"
 export * from "./row"
+export * from "./user"

packages/types/src/documents/app/query.ts

@@ -1 +1,3 @@
-export interface Query {}
+export interface Query {
+  datasourceId: string
+}

packages/types/src/documents/app/role.ts

@@ -1 +1,3 @@
-export interface Role {}
+import { Document } from "./document"
+
+export interface Role extends Document {}

packages/types/src/documents/app/user.ts

@@ -0,0 +1,3 @@
+export interface UserMetadata {
+  roleId: string
+}

packages/types/src/documents/global/user.ts

@@ -1 +1,7 @@
-export interface User {}
+export interface User {
+  roles: UserRoles
+}
+
+export interface UserRoles {
+  [key: string]: string
+}

packages/worker/package.json

@@ -64,6 +64,7 @@
     "server-destroy": "^1.0.1"
   },
   "devDependencies": {
+    "@budibase/types": "^1.0.126-alpha.0",
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",

packages/worker/src/sdk/users/events.ts

@@ -1,4 +1,5 @@
-const { events } = require("@budibase/backend-core")
+import { events } from "@budibase/backend-core"
+import { User, UserRoles } from "@budibase/types"
 
 export const handleDeleteEvents = (user: any) => {
   events.user.deleted(user)
@@ -12,23 +13,31 @@ export const handleDeleteEvents = (user: any) => {
   }
 }
 
-const assignAppRoleEvents = (roles: any, existingRoles: any) => {
+const assignAppRoleEvents = (
+  user: User,
+  roles: UserRoles,
+  existingRoles: UserRoles
+) => {
   for (const [appId, role] of Object.entries(roles)) {
     // app role in existing is not same as new
     if (!existingRoles || existingRoles[appId] !== role) {
-      events.role.assigned(role)
+      events.role.assigned(user, role)
     }
   }
 }
 
-const unassignAppRoleEvents = (roles: any, existingRoles: any) => {
+const unassignAppRoleEvents = (
+  user: User,
+  roles: UserRoles,
+  existingRoles: UserRoles
+) => {
   if (!existingRoles) {
     return
   }
   for (const [appId, role] of Object.entries(existingRoles)) {
     // app role in new is not same as existing
     if (!roles || roles[appId] !== role) {
-      events.role.unassigned(role)
+      events.role.unassigned(user, role)
     }
   }
 }
@@ -37,8 +46,8 @@ const handleAppRoleEvents = (user: any, existingUser: any) => {
   const roles = user.roles
   const existingRoles = existingUser?.roles
 
-  assignAppRoleEvents(roles, existingRoles)
-  unassignAppRoleEvents(roles, existingRoles)
+  assignAppRoleEvents(user, roles, existingRoles)
+  unassignAppRoleEvents(user, roles, existingRoles)
 }
 
 export const handleSaveEvents = (user: any, existingUser: any) => {