Add app db support + app url migration

packages/backend-core/src/migrations/index.js

@@ -2,7 +2,12 @@ const { DEFAULT_TENANT_ID } = require("../constants")
 const { DocumentTypes } = require("../db/constants")
 const { getAllApps } = require("../db/utils")
 const environment = require("../environment")
-const { doInTenant, getTenantIds, getGlobalDBName } = require("../tenancy")
+const {
+  doInTenant,
+  getTenantIds,
+  getGlobalDBName,
+  getTenantId,
+} = require("../tenancy")
 
 exports.MIGRATION_TYPES = {
   GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument
@@ -20,16 +25,18 @@ exports.getMigrationsDoc = async db => {
   }
 }
 
-const runMigration = async (tenantId, CouchDB, migration, options = {}) => {
+const runMigration = async (CouchDB, migration, options = {}) => {
+  const tenantId = getTenantId()
   const migrationType = migration.type
   const migrationName = migration.name
 
   // get the db to store the migration in
   let dbNames
   if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
-    dbNames = [getGlobalDBName(tenantId)]
+    dbNames = [getGlobalDBName()]
   } else if (migrationType === exports.MIGRATION_TYPES.APP) {
-    dbNames = await getAllApps(CouchDB, { all: true })
+    const apps = await getAllApps(CouchDB, migration.opts)
+    dbNames = apps.map(app => app.appId)
   } else {
     throw new Error(
       `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]`
@@ -50,7 +57,7 @@ const runMigration = async (tenantId, CouchDB, migration, options = {}) => {
           options.force[migrationType].includes(migrationName)
         ) {
           console.log(
-            `[Tenant: ${tenantId}] Forcing migration [${migrationName}]`
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
           )
         } else {
           // the migration has already been performed
@@ -59,18 +66,20 @@ const runMigration = async (tenantId, CouchDB, migration, options = {}) => {
       }
 
       console.log(
-        `[Tenant: ${tenantId}] Performing migration: ${migrationName}`
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running`
       )
       // run the migration with tenant context
-      await doInTenant(tenantId, () => migration.fn(db))
-      console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
+      await migration.fn(db)
+      console.log(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+      )
 
       // mark as complete
       doc[migrationName] = Date.now()
       await db.put(doc)
     } catch (err) {
       console.error(
-        `[Tenant: ${tenantId}] Error performing migration: ${migrationName} on db: ${db.name}: `,
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
         err
       )
       throw err
@@ -96,7 +105,9 @@ exports.runMigrations = async (CouchDB, migrations, options = {}) => {
     // for all migrations
     for (const migration of migrations) {
       // run the migration
-      await runMigration(tenantId, CouchDB, migration, options)
+      await doInTenant(tenantId, () =>
+        runMigration(CouchDB, migration, options)
+      )
     }
   }
   console.log("Migrations complete")

packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap

@@ -3,7 +3,7 @@
 exports[`migrations should match snapshot 1`] = `
 Object {
   "_id": "migrations",
-  "_rev": "1-af6c272fe081efafecd2ea49a8fcbb40",
-  "user_email_view_casing": 1487076708000,
+  "_rev": "1-6277abc4e3db950221768e5a2618a059",
+  "test": 1487076708000,
 }
 `;

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,7 +1,7 @@
 require("../../tests/utilities/dbConfig")
 
-const { migrateIfRequired, MIGRATION_DBS, MIGRATIONS, getMigrationsDoc } = require("../index")
-const database = require("../../db")
+const { runMigrations, getMigrationsDoc } = require("../index")
+const CouchDB = require("../../db").getCouch()
 const {
   StaticDatabases,
 } = require("../../db/utils")
@@ -13,8 +13,14 @@ describe("migrations", () => {
 
   const migrationFunction = jest.fn()
 
+  const MIGRATIONS = [{
+    type: "global",
+    name: "test",
+    fn: migrationFunction
+  }]
+
   beforeEach(() => {
-    db = database.getDB(StaticDatabases.GLOBAL.name)
+    db = new CouchDB(StaticDatabases.GLOBAL.name)
   })
 
   afterEach(async () => {
@@ -22,39 +28,29 @@ describe("migrations", () => {
     await db.destroy()
   })
 
-  const validMigration = () => {
-    return migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
+  const migrate = () => {
+    return runMigrations(CouchDB, MIGRATIONS)
   }
 
   it("should run a new migration", async () => {
-    await validMigration()
+    await migrate()
     expect(migrationFunction).toHaveBeenCalled()
+    const doc = await getMigrationsDoc(db)
+    expect(doc.test).toBeDefined()
   })
 
   it("should match snapshot", async () => {
-    await validMigration()
+    await migrate()
     const doc = await getMigrationsDoc(db)
     expect(doc).toMatchSnapshot()
   })
 
   it("should skip a previously run migration", async () => {
-    await validMigration()
-    await validMigration()
+    await migrate()
+    const previousMigrationTime = await getMigrationsDoc(db).test
+    await migrate()
+    const currentMigrationTime = await getMigrationsDoc(db).test
     expect(migrationFunction).toHaveBeenCalledTimes(1)
+    expect(currentMigrationTime).toBe(previousMigrationTime)
   })
-
-  it("should reject an unknown migration name", async () => {
-    expect(async () => { 
-      await migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, "bogus_name", migrationFunction)
-    }).rejects.toThrow()
-    expect(migrationFunction).not.toHaveBeenCalled()
-  })
-
-  it("should reject an unknown database name", async () => {
-    expect(async () => { 
-      await migrateIfRequired("bogus_db", MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
-    }).rejects.toThrow()
-    expect(migrationFunction).not.toHaveBeenCalled()
-  })
-
 })

packages/server/src/api/controllers/application.js

@@ -75,7 +75,7 @@ function getUserRoleId(ctx) {
     : ctx.user.role._id
 }
 
-async function getAppUrl(ctx) {
+exports.getAppUrl = ctx => {
   // construct the url
   let url
   if (ctx.request.body.url) {
@@ -218,7 +218,7 @@ exports.create = async ctx => {
   const apps = await getAllApps(CouchDB, { dev: true })
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name)
-  const url = await getAppUrl(ctx)
+  const url = exports.getAppUrl(ctx)
   checkAppUrl(ctx, apps, url)
 
   const { useTemplate, templateKey, templateString } = ctx.request.body
@@ -281,7 +281,7 @@ exports.update = async ctx => {
   // validation
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name, ctx.params.appId)
-  const url = await getAppUrl(ctx)
+  const url = exports.getAppUrl(ctx)
   checkAppUrl(ctx, apps, url, ctx.params.appId)
 
   const appPackageUpdates = { name, url }

packages/server/src/api/controllers/migrations.js

@@ -1,4 +1,4 @@
-const { migrate } = require("../../migrations")
+const { migrate, MIGRATIONS } = require("../../migrations")
 
 exports.migrate = async ctx => {
   const options = ctx.request.body
@@ -6,3 +6,8 @@ exports.migrate = async ctx => {
   migrate(options)
   ctx.status = 200
 }
+
+exports.fetchDefinitions = async ctx => {
+  ctx.body = MIGRATIONS
+  ctx.status = 200
+}

packages/server/src/api/routes/migrations.js

@@ -3,6 +3,12 @@ const migrationsController = require("../controllers/migrations")
 const router = Router()
 const { internalApi } = require("@budibase/backend-core/auth")
 
-router.post("/api/migrations/run", internalApi, migrationsController.migrate)
+router
+  .post("/api/migrations/run", internalApi, migrationsController.migrate)
+  .get(
+    "/api/migrations/definitions",
+    internalApi,
+    migrationsController.fetchDefinitions
+  )
 
 module.exports = router

packages/server/src/integrations/rest.ts

@@ -43,8 +43,8 @@ const coreFields = {
     enum: Object.values(BodyTypes),
   },
   pagination: {
-    type: DatasourceFieldTypes.OBJECT
-  }
+    type: DatasourceFieldTypes.OBJECT,
+  },
 }
 
 module RestModule {
@@ -178,12 +178,17 @@ module RestModule {
           headers,
         },
         pagination: {
-          cursor: nextCursor
-        }
+          cursor: nextCursor,
+        },
       }
     }
 
-    getUrl(path: string, queryString: string, pagination: PaginationConfig | null, paginationValues: PaginationValues | null): string {
+    getUrl(
+      path: string,
+      queryString: string,
+      pagination: PaginationConfig | null,
+      paginationValues: PaginationValues | null
+    ): string {
       // Add pagination params to query string if required
       if (pagination?.location === "query" && paginationValues) {
         const { pageParam, sizeParam } = pagination
@@ -217,14 +222,22 @@ module RestModule {
       return complete
     }
 
-    addBody(bodyType: string, body: string | any, input: any, pagination: PaginationConfig | null, paginationValues: PaginationValues | null) {
+    addBody(
+      bodyType: string,
+      body: string | any,
+      input: any,
+      pagination: PaginationConfig | null,
+      paginationValues: PaginationValues | null
+    ) {
       if (!input.headers) {
         input.headers = {}
       }
       if (bodyType === BodyTypes.NONE) {
         return input
       }
-      let error, object: any = {}, string = ""
+      let error,
+        object: any = {},
+        string = ""
       try {
         if (body) {
           string = typeof body !== "string" ? JSON.stringify(body) : body
@@ -333,7 +346,7 @@ module RestModule {
         requestBody,
         authConfigId,
         pagination,
-        paginationValues
+        paginationValues,
       } = query
       const authHeaders = this.getAuthHeaders(authConfigId)
 
@@ -352,7 +365,13 @@ module RestModule {
       }
 
       let input: any = { method, headers: this.headers }
-      input = this.addBody(bodyType, requestBody, input, pagination, paginationValues)
+      input = this.addBody(
+        bodyType,
+        requestBody,
+        input,
+        pagination,
+        paginationValues
+      )
 
       this.startTimeMs = performance.now()
       const url = this.getUrl(path, queryString, pagination, paginationValues)

packages/server/src/integrations/s3.ts

@@ -38,7 +38,7 @@ module S3Module {
       signatureVersion: {
         type: "string",
         required: false,
-        default: "v4"
+        default: "v4",
       },
     },
     query: {

packages/server/src/migrations/functions/appUrls.ts

@@ -0,0 +1,25 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+import { getAppUrl } from "../../api/controllers/application"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Add the url to the app metadata if it doesn't exist
+ */
+export const run = async (appDb: any) => {
+  const metadata = await appDb.get(DocumentTypes.APP_METADATA)
+  if (!metadata.url) {
+    const context = {
+      request: {
+        body: {
+          name: metadata.name,
+        },
+      },
+    }
+    metadata.url = getAppUrl(context)
+    console.log(`Adding url to app: ${metadata.url}`)
+  }
+  await appDb.put(metadata)
+}

packages/server/src/migrations/functions/tests/appUrls.spec.js

@@ -0,0 +1,29 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const migration = require("../appUrls")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const CouchDB = config.getCouch()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    const app = await config.createApp("testApp")
+    const appDb = new CouchDB(app.appId)
+    let metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    delete metadata.url
+    await appDb.put(metadata)
+    
+    await migration.run(appDb)
+
+    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    expect(metadata.url).toEqual("/testapp")
+  })
+})

packages/server/src/migrations/functions/tests/quotas1.spec.js

@@ -0,0 +1,27 @@
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const syncApps = jest.fn()
+const syncRows = jest.fn()
+
+jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
+
+const migration = require("../quotas1")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs ", async () => {
+    await migration.run()
+    expect(syncApps).toHaveBeenCalledTimes(1)
+    expect(syncRows).toHaveBeenCalledTimes(1)
+  })
+})

packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -0,0 +1,25 @@
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+
+// mock email view creation
+const coreDb = require("@budibase/backend-core/db")
+const createUserEmailView = jest.fn()
+coreDb.createUserEmailView = createUserEmailView
+
+const migration = require("../userEmailViewCasing")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const globalDb = getGlobalDB()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    await migration.run(globalDb)
+    expect(createUserEmailView).toHaveBeenCalledTimes(1)
+  })
+})

packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js

@@ -1,27 +0,0 @@
-const env = require("../../../../environment")
-const TestConfig = require("../../../../tests/utilities/TestConfiguration")
-
-const syncApps = jest.fn()
-const syncRows = jest.fn()
-
-jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
-jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
-
-const migrations = require("..")
-
-describe("run", () => {
-  let config = new TestConfig(false)
-
-  beforeEach(async () => {
-    await config.init()
-    env._set("USE_QUOTAS", 1)
-  })
-
-  afterAll(config.end)  
-
-  it("runs the required migrations", async () => {
-    await migrations.run()
-    expect(syncApps).toHaveBeenCalledTimes(1)
-    expect(syncRows).toHaveBeenCalledTimes(1)
-  })
-})

packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js

@@ -1,8 +1,8 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
 const syncApps = require("../syncApps")
-const env = require("../../../environment")
+const env = require("../../../../environment")
 
 describe("syncApps", () => {
   let config = new TestConfig(false)

packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js

@@ -1,8 +1,8 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
 const syncRows = require("../syncRows")
-const env = require("../../../environment")
+const env = require("../../../../environment")
 
 describe("syncRows", () => {
   let config = new TestConfig(false)

packages/server/src/migrations/index.ts

@@ -7,10 +7,12 @@ const {
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
 import * as quota1 from "./functions/quotas1"
+import * as appUrls from "./functions/appUrls"
 
 export interface Migration {
   type: string
   name: string
+  opts?: object
   fn: Function
 }
 
@@ -30,7 +32,7 @@ export interface MigrationOptions {
   }
 }
 
-const MIGRATIONS: Migration[] = [
+export const MIGRATIONS: Migration[] = [
   {
     type: MIGRATION_TYPES.GLOBAL,
     name: "user_email_view_casing",
@@ -41,6 +43,12 @@ const MIGRATIONS: Migration[] = [
     name: "quotas_1",
     fn: quota1.run,
   },
+  {
+    type: MIGRATION_TYPES.APP,
+    name: "app_urls",
+    opts: { all: true },
+    fn: appUrls.run,
+  },
 ]
 
 export const migrate = async (options?: MigrationOptions) => {

packages/server/src/tests/utilities/TestConfiguration.js

@@ -49,6 +49,10 @@ class TestConfiguration {
     return this.appId
   }
 
+  getCouch() {
+    return CouchDB
+  }
+
   async _req(config, params, controlFunc) {
     const request = {}
     // fake cookies, we don't need them