Merge branch 'develop' of github.com:Budibase/budibase into lab-day/refactor-app-db

charts/budibase/templates/app-service-deployment.yaml

@@ -108,6 +108,8 @@ spec:
           value: {{ .Values.globals.accountPortalApiKey | quote }}
         - name: COOKIE_DOMAIN
           value: {{ .Values.globals.cookieDomain | quote }}
+        - name: HTTP_MIGRATIONS
+          value: {{ .Values.globals.httpMigrations | quote }}
         image: budibase/apps:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always
         name: bbapps

charts/budibase/values.yaml

@@ -99,6 +99,7 @@ globals:
   accountPortalApiKey: ""
   cookieDomain: ""
   platformUrl: ""
+  httpMigrations: "0"
 
   createSecrets: true # creates an internal API key, JWT secrets and redis password for you
 

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/db.js

@@ -2,4 +2,5 @@ module.exports = {
   ...require("./src/db/utils"),
   ...require("./src/db/constants"),
   ...require("./src/db"),
+  ...require("./src/db/views"),
 }

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/backend-core/src/auth.js

@@ -13,6 +13,7 @@ const {
   appTenancy,
   authError,
   csrf,
+  internalApi,
 } = require("./middleware")
 
 // Strategies
@@ -44,4 +45,5 @@ module.exports = {
   auditLog,
   authError,
   buildCsrfMiddleware: csrf,
+  internalApi,
 }

packages/backend-core/src/middleware/index.js

@@ -7,6 +7,7 @@ const authenticated = require("./authenticated")
 const auditLog = require("./auditLog")
 const tenancy = require("./tenancy")
 const appTenancy = require("./appTenancy")
+const internalApi = require("./internalApi")
 const datasourceGoogle = require("./passport/datasource/google")
 const csrf = require("./csrf")
 
@@ -20,6 +21,7 @@ module.exports = {
   tenancy,
   appTenancy,
   authError,
+  internalApi,
   datasource: {
     google: datasourceGoogle,
   },

packages/backend-core/src/middleware/internalApi.js

@@ -0,0 +1,14 @@
+const env = require("../environment")
+const { Headers } = require("../constants")
+
+/**
+ * API Key only endpoint.
+ */
+module.exports = async (ctx, next) => {
+  const apiKey = ctx.request.headers[Headers.API_KEY]
+  if (apiKey !== env.INTERNAL_API_KEY) {
+    ctx.throw(403, "Unauthorized")
+  }
+
+  return next()
+}

packages/backend-core/src/migrations/index.js

@@ -1,20 +1,17 @@
+const { DEFAULT_TENANT_ID } = require("../constants")
 const { DocumentTypes } = require("../db/constants")
-const { getGlobalDB, getTenantId } = require("../tenancy")
+const { getAllApps } = require("../db/utils")
+const environment = require("../environment")
+const {
+  doInTenant,
+  getTenantIds,
+  getGlobalDBName,
+  getTenantId,
+} = require("../tenancy")
 
-exports.MIGRATION_DBS = {
+exports.MIGRATION_TYPES = {
-  GLOBAL_DB: "GLOBAL_DB",
+  GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument
-}
+  APP: "app", // run per app, recorded in each app db, app db is provided as an argument
-
-exports.MIGRATIONS = {
-  USER_EMAIL_VIEW_CASING: "user_email_view_casing",
-  QUOTAS_1: "quotas_1",
-}
-
-const DB_LOOKUP = {
-  [exports.MIGRATION_DBS.GLOBAL_DB]: [
-    exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
-    exports.MIGRATIONS.QUOTAS_1,
-  ],
 }
 
 exports.getMigrationsDoc = async db => {
@@ -28,40 +25,90 @@ exports.getMigrationsDoc = async db => {
   }
 }
 
-exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
+const runMigration = async (CouchDB, migration, options = {}) => {
   const tenantId = getTenantId()
-  try {
+  const migrationType = migration.type
-    let db
+  const migrationName = migration.name
-    if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
+
-      db = getGlobalDB()
+  // get the db to store the migration in
-    } else {
+  let dbNames
-      throw new Error(`Unrecognised migration db [${migrationDb}]`)
+  if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
-    }
+    dbNames = [getGlobalDBName()]
+  } else if (migrationType === exports.MIGRATION_TYPES.APP) {
+    const apps = await getAllApps(CouchDB, migration.opts)
+    dbNames = apps.map(app => app.appId)
+  } else {
+    throw new Error(
+      `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]`
+    )
+  }
+
+  // run the migration against each db
+  for (const dbName of dbNames) {
+    const db = new CouchDB(dbName)
+    try {
+      const doc = await exports.getMigrationsDoc(db)
+
+      // exit if the migration has been performed already
+      if (doc[migrationName]) {
+        if (
+          options.force &&
+          options.force[migrationType] &&
+          options.force[migrationType].includes(migrationName)
+        ) {
+          console.log(
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
+          )
+        } else {
+          // the migration has already been performed
+          continue
+        }
+      }
 
-    if (!DB_LOOKUP[migrationDb].includes(migrationName)) {
+      console.log(
-      throw new Error(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running`
-        `Unrecognised migration name [${migrationName}] for db [${migrationDb}]`
+      )
+      // run the migration with tenant context
+      await migration.fn(db)
+      console.log(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
       )
-    }
 
-    const doc = await exports.getMigrationsDoc(db)
+      // mark as complete
-    // exit if the migration has been performed
+      doc[migrationName] = Date.now()
-    if (doc[migrationName]) {
+      await db.put(doc)
-      return
+    } catch (err) {
+      console.error(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+        err
+      )
+      throw err
     }
+  }
+}
 
-    console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
+exports.runMigrations = async (CouchDB, migrations, options = {}) => {
-    await migrateFn()
+  console.log("Running migrations")
-    console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
+  let tenantIds
+  if (environment.MULTI_TENANCY) {
+    if (!options.tenantIds || !options.tenantIds.length) {
+      // run for all tenants
+      tenantIds = await getTenantIds()
+    }
+  } else {
+    // single tenancy
+    tenantIds = [DEFAULT_TENANT_ID]
+  }
 
-    // mark as complete
+  // for all tenants
-    doc[migrationName] = Date.now()
+  for (const tenantId of tenantIds) {
-    await db.put(doc)
+    // for all migrations
-  } catch (err) {
+    for (const migration of migrations) {
-    console.error(
+      // run the migration
-      `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
+      await doInTenant(tenantId, () =>
-      err
+        runMigration(CouchDB, migration, options)
-    )
+      )
-    throw err
+    }
   }
+  console.log("Migrations complete")
 }

packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap

@@ -3,7 +3,7 @@
 exports[`migrations should match snapshot 1`] = `
 Object {
   "_id": "migrations",
-  "_rev": "1-af6c272fe081efafecd2ea49a8fcbb40",
+  "_rev": "1-6277abc4e3db950221768e5a2618a059",
-  "user_email_view_casing": 1487076708000,
+  "test": 1487076708000,
 }
 `;

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,7 +1,7 @@
 require("../../tests/utilities/dbConfig")
 
-const { migrateIfRequired, MIGRATION_DBS, MIGRATIONS, getMigrationsDoc } = require("../index")
+const { runMigrations, getMigrationsDoc } = require("../index")
-const database = require("../../db")
+const CouchDB = require("../../db").getCouch()
 const {
   StaticDatabases,
 } = require("../../db/utils")
@@ -13,8 +13,14 @@ describe("migrations", () => {
 
   const migrationFunction = jest.fn()
 
+  const MIGRATIONS = [{
+    type: "global",
+    name: "test",
+    fn: migrationFunction
+  }]
+
   beforeEach(() => {
-    db = database.getDB(StaticDatabases.GLOBAL.name)
+    db = new CouchDB(StaticDatabases.GLOBAL.name)
   })
 
   afterEach(async () => {
@@ -22,39 +28,29 @@ describe("migrations", () => {
     await db.destroy()
   })
 
-  const validMigration = () => {
+  const migrate = () => {
-    return migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
+    return runMigrations(CouchDB, MIGRATIONS)
   }
 
   it("should run a new migration", async () => {
-    await validMigration()
+    await migrate()
     expect(migrationFunction).toHaveBeenCalled()
+    const doc = await getMigrationsDoc(db)
+    expect(doc.test).toBeDefined()
   })
 
   it("should match snapshot", async () => {
-    await validMigration()
+    await migrate()
     const doc = await getMigrationsDoc(db)
     expect(doc).toMatchSnapshot()
   })
 
   it("should skip a previously run migration", async () => {
-    await validMigration()
+    await migrate()
-    await validMigration()
+    const previousMigrationTime = await getMigrationsDoc(db).test
+    await migrate()
+    const currentMigrationTime = await getMigrationsDoc(db).test
     expect(migrationFunction).toHaveBeenCalledTimes(1)
+    expect(currentMigrationTime).toBe(previousMigrationTime)
   })
-
-  it("should reject an unknown migration name", async () => {
-    expect(async () => { 
-      await migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, "bogus_name", migrationFunction)
-    }).rejects.toThrow()
-    expect(migrationFunction).not.toHaveBeenCalled()
-  })
-
-  it("should reject an unknown database name", async () => {
-    expect(async () => { 
-      await migrateIfRequired("bogus_db", MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
-    }).rejects.toThrow()
-    expect(migrationFunction).not.toHaveBeenCalled()
-  })
-
 })

packages/backend-core/src/tenancy/tenancy.js

@@ -148,3 +148,15 @@ exports.isUserInAppTenant = (appId, user = null) => {
   const tenantId = exports.getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
   return tenantId === userTenantId
 }
+
+exports.getTenantIds = async () => {
+  const db = getDB(PLATFORM_INFO_DB)
+  let tenants
+  try {
+    tenants = await db.get(TENANT_DOC)
+  } catch (err) {
+    // if theres an error the doc doesn't exist, no tenants exist
+    return []
+  }
+  return (tenants && tenants.tenantIds) || []
+}

packages/backend-core/src/utils.js

@@ -20,9 +20,6 @@ const { hash } = require("./hashing")
 const userCache = require("./cache/user")
 const env = require("./environment")
 const { getUserSessions, invalidateSessions } = require("./security/sessions")
-const { migrateIfRequired } = require("./migrations")
-const { USER_EMAIL_VIEW_CASING } = require("./migrations").MIGRATIONS
-const { GLOBAL_DB } = require("./migrations").MIGRATION_DBS
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
 
@@ -144,11 +141,6 @@ exports.getGlobalUserByEmail = async email => {
   }
   const db = getGlobalDB()
 
-  await migrateIfRequired(GLOBAL_DB, USER_EMAIL_VIEW_CASING, async () => {
-    // re-create the view with latest changes
-    await createUserEmailView(db)
-  })
-
   try {
     let users = (
       await db.query(`database/${ViewNames.USER_BY_EMAIL}`, {

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -66,10 +66,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.49-alpha.1",
+    "@budibase/bbui": "^1.0.49-alpha.2",
-    "@budibase/client": "^1.0.49-alpha.1",
+    "@budibase/client": "^1.0.49-alpha.2",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.49-alpha.1",
+    "@budibase/string-templates": "^1.0.49-alpha.2",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/pages/builder/_layout.svelte

@@ -61,7 +61,7 @@
       await auth.setInitInfo({ init_template: $params["?template"] })
     }
 
-    await auth.checkAuth()
+    await auth.getSelf()
     await admin.init()
 
     if (useAccountPortal && multiTenancyEnabled) {

packages/builder/src/pages/builder/auth/reset.svelte

@@ -31,7 +31,7 @@
   }
 
   onMount(async () => {
-    await auth.checkAuth()
+    await auth.getSelf()
     await organisation.init()
   })
 </script>

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -337,6 +337,14 @@
             }}
             class="template-card"
           >
+            <a
+              href={item.url}
+              target="_blank"
+              class="external-link"
+              on:click|stopPropagation
+            >
+              <Icon name="LinkOut" size="S" />
+            </a>
             <div class="card-body">
               <div style="color: {item.background}" class="iconAlign">
                 <svg
@@ -518,6 +526,7 @@
     border: 1px solid var(--spectrum-global-color-gray-300);
     cursor: pointer;
     display: flex;
+    position: relative;
   }
 
   .template-card:hover {
@@ -528,6 +537,18 @@
     align-items: center;
     padding: 12px;
   }
+
+  .external-link {
+    position: absolute;
+    top: 5px;
+    right: 5px;
+    color: var(--spectrum-global-color-gray-300);
+    z-index: 99;
+  }
+  .external-link:hover {
+    color: var(--spectrum-global-color-gray-500);
+  }
+
   .iconAlign {
     padding: 0 0 0 var(--spacing-m);
     display: inline-block;

packages/builder/src/stores/portal/auth.js

@@ -108,11 +108,7 @@ export function createAuthStore() {
     return json
   }
 
-  return {
+  const actions = {
-    subscribe: store.subscribe,
-    setOrganisation,
-    getInitInfo,
-    setInitInfo,
     checkQueryString: async () => {
       const urlParams = new URLSearchParams(window.location.search)
       if (urlParams.has("tenantId")) {
@@ -123,7 +119,7 @@ export function createAuthStore() {
     setOrg: async tenantId => {
       await setOrganisation(tenantId)
     },
-    checkAuth: async () => {
+    getSelf: async () => {
       const response = await api.get("/api/global/users/self")
       if (response.status !== 200) {
         setUser(null)
@@ -138,13 +134,12 @@ export function createAuthStore() {
         `/api/global/auth/${tenantId}/login`,
         creds
       )
-      const json = await response.json()
       if (response.status === 200) {
-        setUser(json.user)
+        await actions.getSelf()
       } else {
+        const json = await response.json()
         throw new Error(json.message ? json.message : "Invalid credentials")
       }
-      return json
     },
     logout: async () => {
       const response = await api.post(`/api/global/auth/logout`)
@@ -197,6 +192,14 @@ export function createAuthStore() {
       await response.json()
     },
   }
+
+  return {
+    subscribe: store.subscribe,
+    setOrganisation,
+    getInitInfo,
+    setInitInfo,
+    ...actions,
+  }
 }
 
 export const auth = createAuthStore()

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.49-alpha.1",
+    "@budibase/bbui": "^1.0.49-alpha.2",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.49-alpha.1",
+    "@budibase/string-templates": "^1.0.49-alpha.2",
     "regexparam": "^1.3.0",
     "rollup-plugin-polyfill-node": "^0.8.0",
     "shortid": "^2.2.15",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.49-alpha.1",
+    "@budibase/backend-core": "^1.0.49-alpha.2",
-    "@budibase/client": "^1.0.49-alpha.1",
+    "@budibase/client": "^1.0.49-alpha.2",
-    "@budibase/string-templates": "^1.0.49-alpha.1",
+    "@budibase/string-templates": "^1.0.49-alpha.2",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/application.js

@@ -77,7 +77,7 @@ function getUserRoleId(ctx) {
     : ctx.user.role._id
 }
 
-async function getAppUrl(ctx) {
+exports.getAppUrl = ctx => {
   // construct the url
   let url
   if (ctx.request.body.url) {
@@ -221,7 +221,7 @@ exports.create = async ctx => {
   const apps = await getAllApps({ dev: true })
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name)
-  const url = await getAppUrl(ctx)
+  const url = exports.getAppUrl(ctx)
   checkAppUrl(ctx, apps, url)
 
   const { useTemplate, templateKey, templateString } = ctx.request.body
@@ -288,7 +288,7 @@ exports.update = async ctx => {
   if (name) {
     checkAppName(ctx, apps, name, ctx.params.appId)
   }
-  const url = await getAppUrl(ctx)
+  const url = await exports.getAppUrl(ctx)
   if (url) {
     checkAppUrl(ctx, apps, url, ctx.params.appId)
     ctx.request.body.url = url

packages/server/src/api/controllers/migrations.js

@@ -0,0 +1,13 @@
+const { migrate, MIGRATIONS } = require("../../migrations")
+
+exports.migrate = async ctx => {
+  const options = ctx.request.body
+  // don't await as can take a while, just return
+  migrate(options)
+  ctx.status = 200
+}
+
+exports.fetchDefinitions = async ctx => {
+  ctx.body = MIGRATIONS
+  ctx.status = 200
+}

packages/server/src/api/controllers/query/import/index.ts

@@ -34,9 +34,7 @@ export class RestImporter {
     return this.source.getInfo()
   }
 
-  importQueries = async (
+  importQueries = async (datasourceId: string): Promise<ImportResult> => {
-    datasourceId: string
-  ): Promise<ImportResult> => {
     // constuct the queries
     let queries = await this.source.getQueries(datasourceId)
 

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -26,7 +26,7 @@ import {
   RelationshipTypes,
 } from "../../../constants"
 import { breakExternalTableId, isSQL } from "../../../integrations/utils"
-import { processObjectSync }  from "@budibase/string-templates"
+import { processObjectSync } from "@budibase/string-templates"
 // @ts-ignore
 import { cloneDeep } from "lodash/fp"
 import { processFormulas } from "../../../utilities/rowProcessor/utils"
@@ -216,11 +216,7 @@ module External {
     private datasource: Datasource
     private tables: { [key: string]: Table } = {}
 
-    constructor(
+    constructor(operation: Operation, tableId: string, datasource: Datasource) {
-      operation: Operation,
-      tableId: string,
-      datasource: Datasource
-    ) {
       this.operation = operation
       this.tableId = tableId
       this.datasource = datasource
@@ -272,7 +268,9 @@ module External {
           newRow[key] = row[key]
           continue
         }
-        const { tableName: linkTableName } = breakExternalTableId(field?.tableId)
+        const { tableName: linkTableName } = breakExternalTableId(
+          field?.tableId
+        )
         // table has to exist for many to many
         if (!linkTableName || !this.tables[linkTableName]) {
           continue
@@ -559,7 +557,10 @@ module External {
       )) {
         const table: Table | undefined = this.getTable(tableId)
         // if its not the foreign key skip it, nothing to do
-        if (!table || (table.primary && table.primary.indexOf(colName) !== -1)) {
+        if (
+          !table ||
+          (table.primary && table.primary.indexOf(colName) !== -1)
+        ) {
           continue
         }
         for (let row of rows) {

packages/server/src/api/routes/index.js

@@ -24,6 +24,7 @@ const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
 const cloudRoutes = require("./cloud")
+const migrationRoutes = require("./migrations")
 
 exports.mainRoutes = [
   authRoutes,
@@ -53,6 +54,7 @@ exports.mainRoutes = [
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
   rowRoutes,
+  migrationRoutes,
 ]
 
 exports.staticRoutes = staticRoutes

packages/server/src/api/routes/migrations.js

@@ -0,0 +1,14 @@
+const Router = require("@koa/router")
+const migrationsController = require("../controllers/migrations")
+const router = Router()
+const { internalApi } = require("@budibase/backend-core/auth")
+
+router
+  .post("/api/migrations/run", internalApi, migrationsController.migrate)
+  .get(
+    "/api/migrations/definitions",
+    internalApi,
+    migrationsController.fetchDefinitions
+  )
+
+module.exports = router

packages/server/src/app.ts

@@ -1,7 +1,7 @@
 // need to load environment first
 import { ExtendableContext } from "koa"
 
-const env = require("./environment")
+import * as env from "./environment"
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")
@@ -16,6 +16,7 @@ const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
 const redis = require("./utilities/redis")
+import * as migrations from "./migrations"
 
 const app = new Koa()
 
@@ -84,13 +85,25 @@ module.exports = server.listen(env.PORT || 0, async () => {
   await automations.init()
 })
 
-process.on("uncaughtException", err => {
+const shutdown = () => {
-  console.error(err)
   server.close()
   server.destroy()
+}
+
+process.on("uncaughtException", err => {
+  console.error(err)
+  shutdown()
 })
 
 process.on("SIGTERM", () => {
-  server.close()
+  shutdown()
-  server.destroy()
 })
+
+// run migrations on startup if not done via http
+// not recommended in a clustered environment
+if (!env.HTTP_MIGRATIONS) {
+  migrations.migrate().catch(err => {
+    console.error("Error performing migrations. Exiting.\n", err)
+    shutdown()
+  })
+}

packages/server/src/environment.js

@@ -44,6 +44,7 @@ module.exports = {
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
+  HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   // environment
   NODE_ENV: process.env.NODE_ENV,
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,

packages/server/src/middleware/usageQuota.js

@@ -4,7 +4,6 @@ const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-const migration = require("../migrations/usageQuotas")
 const { getAppDB } = require("@budibase/backend-core/context")
 
 // currently only counting new writes and deletes
@@ -74,7 +73,6 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await migration.run()
     await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)

packages/server/src/migrations/functions/appUrls.ts

@@ -0,0 +1,25 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+import { getAppUrl } from "../../api/controllers/application"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Add the url to the app metadata if it doesn't exist
+ */
+export const run = async (appDb: any) => {
+  const metadata = await appDb.get(DocumentTypes.APP_METADATA)
+  if (!metadata.url) {
+    const context = {
+      request: {
+        body: {
+          name: metadata.name,
+        },
+      },
+    }
+    metadata.url = getAppUrl(context)
+    console.log(`Adding url to app: ${metadata.url}`)
+  }
+  await appDb.put(metadata)
+}

packages/server/src/migrations/functions/quotas1.ts

@@ -0,0 +1,20 @@
+import { runQuotaMigration } from "./usageQuotas"
+import * as syncApps from "./usageQuotas/syncApps"
+import * as syncRows from "./usageQuotas/syncRows"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Synchronise the app and row quotas to the state of the db after it was
+ * discovered that the quota resets were still in place and the row quotas
+ * weren't being decremented correctly.
+ */
+
+export const run = async () => {
+  await runQuotaMigration(async () => {
+    await syncApps.run()
+    await syncRows.run()
+  })
+}

packages/server/src/migrations/functions/tests/appUrls.spec.js

@@ -0,0 +1,29 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const migration = require("../appUrls")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const CouchDB = config.getCouch()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    const app = await config.createApp("testApp")
+    const appDb = new CouchDB(app.appId)
+    let metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    delete metadata.url
+    await appDb.put(metadata)
+    
+    await migration.run(appDb)
+
+    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    expect(metadata.url).toEqual("/testapp")
+  })
+})

packages/server/src/migrations/tests/usageQuotas/index.spec.js → packages/server/src/migrations/functions/tests/quotas1.spec.js

@@ -4,10 +4,10 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const syncApps = jest.fn()
 const syncRows = jest.fn()
 
-jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
-jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
+jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
 
-const migrations = require("../../usageQuotas")
+const migration = require("../quotas1")
 
 describe("run", () => {
   let config = new TestConfig(false)
@@ -19,8 +19,8 @@ describe("run", () => {
 
   afterAll(config.end)  
 
-  it("runs the required migrations", async () => {
+  it("runs ", async () => {
-    await migrations.run()
+    await migration.run()
     expect(syncApps).toHaveBeenCalledTimes(1)
     expect(syncRows).toHaveBeenCalledTimes(1)
   })

packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -0,0 +1,25 @@
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+
+// mock email view creation
+const coreDb = require("@budibase/backend-core/db")
+const createUserEmailView = jest.fn()
+coreDb.createUserEmailView = createUserEmailView
+
+const migration = require("../userEmailViewCasing")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const globalDb = getGlobalDB()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    await migration.run(globalDb)
+    expect(createUserEmailView).toHaveBeenCalledTimes(1)
+  })
+})

packages/server/src/migrations/functions/usageQuotas/index.ts

@@ -0,0 +1,8 @@
+const { useQuotas } = require("../../../utilities/usageQuota")
+
+export const runQuotaMigration = async (migration: Function) => {
+  if (!useQuotas()) {
+    return
+  }
+  await migration()
+}

packages/server/src/migrations/usageQuotas/syncApps.js → packages/server/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,8 +1,10 @@
-const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+// @ts-ignore
-const { getAllApps } = require("@budibase/backend-core/db")
+import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+// @ts-ignore
+import { getAllApps } from "@budibase/backend-core/db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
 
-exports.run = async () => {
+export const run = async () => {
   const db = getGlobalDB()
   // get app count
   const devApps = await getAllApps({ dev: true })

packages/server/src/migrations/usageQuotas/syncRows.js → packages/server/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,13 +1,15 @@
-const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+// @ts-ignore
-const { getAllApps } = require("@budibase/backend-core/db")
+import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+// @ts-ignore
-const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+import { getAllApps } from "@budibase/backend-core/db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 
-exports.run = async () => {
+export const run = async () => {
   const db = getGlobalDB()
   // get all rows in all apps
   const allApps = await getAllApps({ all: true })
-  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
   const rows = await getUniqueRows(appIds)
   const rowCount = rows ? rows.length : 0
 

packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js → packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js

@@ -1,8 +1,8 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
-const syncApps = require("../../usageQuotas/syncApps")
+const syncApps = require("../syncApps")
-const env = require("../../../environment")
+const env = require("../../../../environment")
 
 describe("syncApps", () => {
   let config = new TestConfig(false)

packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js → packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js

@@ -1,8 +1,8 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
-const syncRows = require("../../usageQuotas/syncRows")
+const syncRows = require("../syncRows")
-const env = require("../../../environment")
+const env = require("../../../../environment")
 
 describe("syncRows", () => {
   let config = new TestConfig(false)

packages/server/src/migrations/functions/userEmailViewCasing.ts

@@ -0,0 +1,13 @@
+const { createUserEmailView } = require("@budibase/backend-core/db")
+
+/**
+ * Date:
+ * October 2021
+ *
+ * Description:
+ * Recreate the user email view to include latest changes i.e. lower casing the email address
+ */
+
+export const run = async (db: any) => {
+  await createUserEmailView(db)
+}

packages/server/src/migrations/index.ts

@@ -0,0 +1,56 @@
+import CouchDB from "../db"
+const {
+  MIGRATION_TYPES,
+  runMigrations,
+} = require("@budibase/backend-core/migrations")
+
+// migration functions
+import * as userEmailViewCasing from "./functions/userEmailViewCasing"
+import * as quota1 from "./functions/quotas1"
+import * as appUrls from "./functions/appUrls"
+
+export interface Migration {
+  type: string
+  name: string
+  opts?: object
+  fn: Function
+}
+
+/**
+ * e.g.
+ * {
+ *   tenantIds: ['bb'],
+ *   force: {
+ *    global: ['quota_1']
+ *   }
+ * }
+ */
+export interface MigrationOptions {
+  tenantIds?: string[]
+  forced?: {
+    [type: string]: string[]
+  }
+}
+
+export const MIGRATIONS: Migration[] = [
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "user_email_view_casing",
+    fn: userEmailViewCasing.run,
+  },
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "quotas_1",
+    fn: quota1.run,
+  },
+  {
+    type: MIGRATION_TYPES.APP,
+    name: "app_urls",
+    opts: { all: true },
+    fn: appUrls.run,
+  },
+]
+
+export const migrate = async (options?: MigrationOptions) => {
+  await runMigrations(CouchDB, MIGRATIONS, options)
+}

packages/server/src/migrations/usageQuotas/index.js

@@ -1,24 +0,0 @@
-const {
-  MIGRATIONS,
-  MIGRATION_DBS,
-  migrateIfRequired,
-} = require("@budibase/backend-core/migrations")
-const { useQuotas } = require("../../utilities/usageQuota")
-const syncApps = require("./syncApps")
-const syncRows = require("./syncRows")
-
-exports.run = async () => {
-  if (!useQuotas()) {
-    return
-  }
-
-  // Jan 2022
-  await migrateIfRequired(
-    MIGRATION_DBS.GLOBAL_DB,
-    MIGRATIONS.QUOTAS_1,
-    async () => {
-      await syncApps.run()
-      await syncRows.run()
-    }
-  )
-}

packages/server/src/tests/utilities/TestConfiguration.js

@@ -51,6 +51,10 @@ class TestConfiguration {
     return this.appId
   }
 
+  getCouch() {
+    return CouchDB
+  }
+
   async _req(config, params, controlFunc) {
     const request = {}
     // fake cookies, we don't need them

packages/server/src/utilities/usageQuota/index.js

@@ -52,6 +52,7 @@ exports.getUsageQuotaDoc = async db => {
  * Given a specified tenantId this will add to the usage object for the specified property.
  * @param {string} property The property which is to be added to (within the nested usageQuota object).
  * @param {number} usage The amount (this can be negative) to adjust the number by.
+ * @param {object} opts optional - options such as dryRun, to check what update will do.
  * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
  * also been reset after this call.
  */

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -29,8 +29,8 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.49-alpha.1",
+    "@budibase/backend-core": "^1.0.49-alpha.2",
-    "@budibase/string-templates": "^1.0.49-alpha.1",
+    "@budibase/string-templates": "^1.0.49-alpha.2",
     "@koa/router": "^8.0.0",
     "@sentry/node": "^6.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",

packages/worker/src/api/controllers/global/auth.js

@@ -74,10 +74,7 @@ async function authInternal(ctx, user, err = null, info = null) {
 exports.authenticate = async (ctx, next) => {
   return passport.authenticate("local", async (err, user, info) => {
     await authInternal(ctx, user, err, info)
-
+    ctx.status = 200
-    delete user.token
-
-    ctx.body = { user }
   })(ctx, next)
 }