Merge pull request #3418 from Budibase/fix/package-deps

Dependency audit/upgrade

.github/workflows/budibase_ci.yml

@@ -18,7 +18,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x]
+        node-version: [14.x]
 
     steps:
     - uses: actions/checkout@v2

package.json

@@ -59,6 +59,7 @@
     "mode:self": "yarn env:selfhost:enable && yarn env:multi:disable && yarn env:account:disable",
     "mode:cloud": "yarn env:selfhost:disable && yarn env:multi:enable && yarn env:account:disable",
     "mode:account": "yarn mode:cloud && yarn env:account:enable",
+    "security:audit": "node scripts/audit.js",
     "postinstall": "husky install"
   }
 }

packages/builder/package.json

@@ -9,7 +9,7 @@
     "test": "jest",
     "test:watch": "jest --watchAll",
     "dev:builder": "routify -c dev:vite",
-    "dev:vite": "vite",
+    "dev:vite": "vite --host 0.0.0.0",
     "rollup": "rollup -c -w",
     "cy:setup": "node ./cypress/setup.js",
     "cy:run": "cypress run",
@@ -91,7 +91,7 @@
     "@babel/runtime": "^7.13.10",
     "@rollup/plugin-replace": "^2.4.2",
     "@roxi/routify": "2.18.0",
-    "@sveltejs/vite-plugin-svelte": "^1.0.0-next.5",
+    "@sveltejs/vite-plugin-svelte": "1.0.0-next.19",
     "@testing-library/jest-dom": "^5.11.10",
     "@testing-library/svelte": "^3.0.0",
     "babel-jest": "^26.6.3",

packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte

@@ -69,6 +69,7 @@
     ({ _id }) => _id === $views.selected?.tableId
   )
   $: fields = viewTable && Object.keys(viewTable.schema)
+  $: schema = viewTable && viewTable.schema ? viewTable.schema : {}
 
   function saveView() {
     views.save(view)
@@ -90,29 +91,29 @@
 
   function isMultipleChoice(field) {
     return (
-      viewTable.schema[field]?.constraints?.inclusion?.length ||
+      schema[field]?.constraints?.inclusion?.length ||
-      viewTable.schema[field]?.type === "boolean"
+      schema[field]?.type === "boolean"
     )
   }
 
   function fieldOptions(field) {
-    return viewTable.schema[field]?.type === "options"
+    return schema[field]?.type === "options"
-      ? viewTable.schema[field]?.constraints.inclusion
+      ? schema[field]?.constraints.inclusion
       : [true, false]
   }
 
   function isDate(field) {
-    return viewTable.schema[field]?.type === "datetime"
+    return schema[field]?.type === "datetime"
   }
 
   function isNumber(field) {
-    return viewTable.schema[field]?.type === "number"
+    return schema[field]?.type === "number"
   }
 
   const fieldChanged = filter => ev => {
     // Reset if type changed
-    const oldType = viewTable.schema[filter.key]?.type
+    const oldType = schema[filter.key]?.type
-    const newType = viewTable.schema[ev.detail]?.type
+    const newType = schema[ev.detail]?.type
     if (filter.key && ev.detail && oldType !== newType) {
       filter.value = ""
     }

packages/builder/vite.config.js

@@ -1,4 +1,4 @@
-import svelte from "@sveltejs/vite-plugin-svelte"
+import { svelte } from "@sveltejs/vite-plugin-svelte"
 import replace from "@rollup/plugin-replace"
 
 import path from "path"
@@ -6,6 +6,11 @@ import path from "path"
 export default ({ mode }) => {
   const isProduction = mode === "production"
   return {
+    server: {
+      fs: {
+        strict: false,
+      },
+    },
     base: "/builder/",
     build: {
       minify: isProduction,

packages/server/package.json

@@ -71,6 +71,8 @@
     "@budibase/auth": "^0.9.185-alpha.9",
     "@budibase/client": "^0.9.185-alpha.9",
     "@budibase/string-templates": "^0.9.185-alpha.9",
+    "@bull-board/api": "^3.7.0",
+    "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -80,7 +82,6 @@
     "aws-sdk": "^2.767.0",
     "bcryptjs": "2.4.3",
     "bull": "^3.22.4",
-    "bull-board": "^2.0.1",
     "chmodr": "1.2.0",
     "csvtojson": "2.0.10",
     "dotenv": "8.2.0",
@@ -139,7 +140,6 @@
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
-    "express": "^4.17.1",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
     "prettier": "^2.3.1",

packages/server/src/api/controllers/application.js

@@ -46,6 +46,7 @@ const {
 const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
 const { syncGlobalUsers } = require("./user")
 const { app: appCache } = require("@budibase/auth/cache")
+const { cleanupAutomations } = require("../../automations/utils")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -319,6 +320,9 @@ exports.delete = async ctx => {
   if (!env.isTest() && !ctx.query.unpublish) {
     await deleteApp(ctx.params.appId)
   }
+  if (ctx.query && ctx.query.unpublish) {
+    await cleanupAutomations(ctx.params.appId)
+  }
   // make sure the app/role doesn't stick around after the app has been deleted
   await removeAppFromUserRoles(ctx, ctx.params.appId)
   await appCache.invalidateAppMetadata(ctx.params.appId)

packages/server/src/app.ts

@@ -41,16 +41,8 @@ app.use(
 )
 
 if (!env.isTest()) {
-  const bullApp = bullboard.init()
+  const plugin = bullboard.init()
-  app.use(async (ctx: ExtendableContext, next: () => any) => {
+  app.use(plugin)
-    if (ctx.path.startsWith(bullboard.pathPrefix)) {
-      ctx.status = 200
-      ctx.respond = false
-      bullApp(ctx.req, ctx.res)
-    } else {
-      await next()
-    }
-  })
 }
 
 app.context.eventEmitter = eventEmitter

packages/server/src/automations/bullboard.js

@@ -1,6 +1,6 @@
-const { createBullBoard } = require("bull-board")
+const { createBullBoard } = require("@bull-board/api")
-const { BullAdapter } = require("bull-board/bullAdapter")
+const { BullAdapter } = require("@bull-board/api/bullAdapter")
-const express = require("express")
+const { KoaAdapter } = require("@bull-board/koa")
 const env = require("../environment")
 const Queue = env.isTest()
   ? require("../utilities/queue/inMemoryQueue")
@@ -19,7 +19,7 @@ async function cleanup() {
   await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
 }
 
-exports.pathPrefix = "/bulladmin"
+const PATH_PREFIX = "/bulladmin"
 
 exports.init = () => {
   // cleanup the events every 5 minutes
@@ -30,17 +30,19 @@ exports.init = () => {
       console.error(`Unable to cleanup automation queue initially - ${err}`)
     })
   }
-  const expressApp = express()
   // Set up queues for bull board admin
   const queues = [automationQueue]
   const adapters = []
+  const serverAdapter = new KoaAdapter()
   for (let queue of queues) {
     adapters.push(new BullAdapter(queue))
   }
-  const { router } = createBullBoard(adapters)
+  createBullBoard({
-
+    queues: adapters,
-  expressApp.use(exports.pathPrefix, router)
+    serverAdapter,
-  return expressApp
+  })
+  serverAdapter.setBasePath(PATH_PREFIX)
+  return serverAdapter.registerPlugin()
 }
 
 exports.queue = automationQueue

packages/server/src/automations/utils.js

@@ -163,3 +163,12 @@ exports.checkForWebhooks = async ({ appId, oldAuto, newAuto }) => {
   }
   return newAuto
 }
+
+/**
+ * When removing an app/unpublishing it need to make sure automations are cleaned up (cron).
+ * @param appId {string} the app that is being removed.
+ * @return {Promise<void>} clean is complete if this succeeds.
+ */
+exports.cleanupAutomations = async appId => {
+  await exports.disableAllCrons(appId)
+}

scripts/audit.js

@@ -0,0 +1,54 @@
+const fs = require("fs")
+const { join } = require("path")
+const { spawnSync } =require("child_process")
+
+const DONT_RUN_PKG = ["bbui"]
+const PACKAGES_PATH = join(__dirname, "..", "packages")
+
+function getPackages() {
+  return fs.readdirSync(PACKAGES_PATH)
+}
+
+function deleteFile(path) {
+  try {
+    fs.unlinkSync(path)
+  } catch (err) {
+    // don't error, it just doesn't exist
+  }
+}
+
+function removeModules(path) {
+  if (fs.existsSync(path)) {
+    fs.rmdirSync(path, { recursive: true })
+  }
+}
+
+function executeInPackage(packageName) {
+  if (DONT_RUN_PKG.includes(packageName)) {
+    return
+  }
+  const dir = join(PACKAGES_PATH, packageName)
+  if (!fs.existsSync(join(dir, "package.json"))) {
+    console.error(`SKIPPING ${packageName} directory, no package.json`)
+    return
+  }
+  const packageLockLoc = join(dir, "package-lock.json")
+  const modulesLoc = join(dir, "node_modules")
+  deleteFile(join(dir, "yarn.lock"))
+  deleteFile(packageLockLoc)
+  removeModules(modulesLoc)
+  const opts = { cwd: dir, stdio: "inherit", shell: true }
+  spawnSync("npm", ["i", "--package-lock-only"], opts)
+  spawnSync("npm", ["audit", "fix"], opts)
+  spawnSync("yarn", ["import"], opts)
+  deleteFile(packageLockLoc)
+  removeModules(modulesLoc)
+}
+
+const packages = getPackages()
+for (let pkg of packages) {
+  executeInPackage(pkg)
+}
+
+spawnSync("yarn", ["bootstrap"], { cwd: join(__dirname, ".."), stdio: "inherit", shell: true })
+