Fixing up middleware to handle uploads, views, records, automation runs and users.

packages/server/src/api/controllers/auth.js

@@ -3,7 +3,7 @@ const CouchDB = require("../../db")
 const ClientDb = require("../../db/clientDb")
 const bcrypt = require("../../utilities/bcrypt")
 const environment = require("../../environment")
-const { apiKeyTable } = require("../../db/dynamoClient")
+const { getAPIKey } = require("../../utilities/usageQuota")
 const { generateUserID } = require("../../db/utils")
 
 exports.authenticate = async ctx => {
@@ -55,7 +55,7 @@ exports.authenticate = async ctx => {
     }
     // if in cloud add the user api key
     if (environment.CLOUD) {
-      payload.apiKey = await apiKeyTable.get({ primary: ctx.user.appId })
+      payload.apiKey = getAPIKey(ctx.user.appId)
     }
 
     const token = jwt.sign(payload, ctx.config.jwtSecret, {

packages/server/src/api/controllers/automation.js

@@ -33,6 +33,7 @@ function cleanAutomationInputs(automation) {
 exports.create = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   let automation = ctx.request.body
+  automation.appId = ctx.user.appId
 
   automation._id = generateAutomationID()
 
@@ -54,6 +55,7 @@ exports.create = async function(ctx) {
 exports.update = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   let automation = ctx.request.body
+  automation.appId = ctx.user.appId
 
   automation = cleanAutomationInputs(automation)
   const response = await db.put(automation)

packages/server/src/api/routes/model.js

@@ -1,7 +1,6 @@
 const Router = require("@koa/router")
 const modelController = require("../controllers/model")
 const authorized = require("../../middleware/authorized")
-const usage = require("../../middleware/usageQuota")
 const { BUILDER, READ_MODEL } = require("../../utilities/accessLevels")
 
 const router = Router()
@@ -13,7 +12,7 @@ router
     authorized(READ_MODEL, ctx => ctx.params.id),
     modelController.find
   )
-  .post("/api/models", authorized(BUILDER), usage, modelController.save)
+  .post("/api/models", authorized(BUILDER), modelController.save)
   .post(
     "/api/models/csv/validate",
     authorized(BUILDER),
@@ -22,7 +21,6 @@ router
   .delete(
     "/api/models/:modelId/:revId",
     authorized(BUILDER),
-    usage,
     modelController.destroy
   )
 

packages/server/src/api/routes/static.js

@@ -4,6 +4,7 @@ const { budibaseTempDir } = require("../../utilities/budibaseDir")
 const env = require("../../environment")
 const authorized = require("../../middleware/authorized")
 const { BUILDER } = require("../../utilities/accessLevels")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -28,7 +29,7 @@ router
     authorized(BUILDER),
     controller.performLocalFileProcessing
   )
-  .post("/api/attachments/upload", controller.uploadFile)
+  .post("/api/attachments/upload", usage, controller.uploadFile)
   .get("/componentlibrary", controller.serveComponentLibrary)
   .get("/assets/:file*", controller.serveAppAsset)
   .get("/attachments/:file*", controller.serveAttachment)

packages/server/src/api/routes/user.js

@@ -2,6 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/user")
 const authorized = require("../../middleware/authorized")
 const { USER_MANAGEMENT, LIST_USERS } = require("../../utilities/accessLevels")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -9,10 +10,11 @@ router
   .get("/api/users", authorized(LIST_USERS), controller.fetch)
   .get("/api/users/:username", authorized(USER_MANAGEMENT), controller.find)
   .put("/api/users/", authorized(USER_MANAGEMENT), controller.update)
-  .post("/api/users", authorized(USER_MANAGEMENT), controller.create)
+  .post("/api/users", authorized(USER_MANAGEMENT), usage, controller.create)
   .delete(
     "/api/users/:username",
     authorized(USER_MANAGEMENT),
+    usage,
     controller.destroy
   )
 

packages/server/src/api/routes/view.js

@@ -3,6 +3,7 @@ const viewController = require("../controllers/view")
 const recordController = require("../controllers/record")
 const authorized = require("../../middleware/authorized")
 const { BUILDER, READ_VIEW } = require("../../utilities/accessLevels")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -13,8 +14,13 @@ router
     recordController.fetchView
   )
   .get("/api/views", authorized(BUILDER), viewController.fetch)
-  .delete("/api/views/:viewName", authorized(BUILDER), viewController.destroy)
-  .post("/api/views", authorized(BUILDER), viewController.save)
+  .delete(
+    "/api/views/:viewName",
+    authorized(BUILDER),
+    usage,
+    viewController.destroy
+  )
+  .post("/api/views", authorized(BUILDER), usage, viewController.save)
   .post("/api/views/export", authorized(BUILDER), viewController.exportView)
   .get(
     "/api/views/export/download/:fileName",

packages/server/src/automations/index.js

@@ -3,6 +3,7 @@ const actions = require("./actions")
 const environment = require("../environment")
 const workerFarm = require("worker-farm")
 const singleThread = require("./thread")
+const { getAPIKey, update, Properties } = require("../utilities/usageQuota")
 
 let workers = workerFarm(require.resolve("./thread"))
 
@@ -18,16 +19,32 @@ function runWorker(job) {
   })
 }
 
+async function updateQuota(automation) {
+  const appId = automation.appId
+  const apiKey = await getAPIKey(appId)
+  // this will fail, causing automation to escape if limits reached
+  await update(apiKey, Properties.AUTOMATION, 1)
+}
+
 /**
  * This module is built purely to kick off the worker farm and manage the inputs/outputs
  */
 module.exports.init = function() {
   actions.init().then(() => {
     triggers.automationQueue.process(async job => {
-      if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
-        await runWorker(job)
-      } else {
-        await singleThread(job)
+      try {
+        if (environment.CLOUD) {
+          await updateQuota(job.data.automation)
+        }
+        if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
+          await runWorker(job)
+        } else {
+          await singleThread(job)
+        }
+      } catch (err) {
+        console.error(
+          `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
+        )
       }
     })
   })

packages/server/src/middleware/usageQuota.js

@@ -1,9 +1,5 @@
 const CouchDB = require("../db")
-const environment = require("../environment")
-const { apiKeyTable } = require("../db/dynamoClient")
-
-// a normalised month in milliseconds
-const QUOTA_RESET = 2592000000
+const usageQuota = require("../utilities/usageQuota")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -12,36 +8,32 @@ const METHOD_MAP = {
 }
 
 const DOMAIN_MAP = {
-  models: "model",
-  records: "record",
+  records: usageQuota.Properties.RECORD,
+  upload: usageQuota.Properties.UPLOAD,
+  views: usageQuota.Properties.VIEW,
+  users: usageQuota.Properties.USER,
+  // this will not be updated by endpoint calls
+  // instead it will be updated by triggers
+  automationRuns: usageQuota.Properties.AUTOMATION,
 }
 
-function buildUpdateParams(key, property, usage) {
-  return {
-    primary: key,
-    condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now",
-    expression: "ADD #quota.#prop :usage",
-    names: {
-      "#quota": "usageQuota",
-      "#prop": property,
-      "#limits": "limits",
-      "#quotaReset": "quotaReset",
-    },
-    values: {
-      ":usage": usage,
-      ":now": Date.now(),
-    },
+function getProperty(url) {
+  for (let domain of Object.keys(DOMAIN_MAP)) {
+    if (url.indexOf(domain) !== -1) {
+      return DOMAIN_MAP[domain]
+    }
   }
 }
 
 module.exports = async (ctx, next) => {
   const db = new CouchDB(ctx.user.instanceId)
-  const usage = METHOD_MAP[ctx.req.method]
-  const domainParts = ctx.req.url.split("/")
-  const property = DOMAIN_MAP[domainParts[domainParts.length - 1]]
+  let usage = METHOD_MAP[ctx.req.method]
+  const property = getProperty(ctx.req.url)
+  console.log(ctx.req.url)
   if (usage == null || property == null) {
     return next()
   }
+  console.log(`${usage} to ${property}`)
   // post request could be a save of a pre-existing entry
   if (ctx.request.body && ctx.request.body._id) {
     try {
@@ -52,26 +44,17 @@ module.exports = async (ctx, next) => {
       return
     }
   }
-  // don't try validate in builder
-  if (!environment.CLOUD) {
-    return next()
+  // update usage for uploads to be the total size
+  if (property === usageQuota.Properties.UPLOAD) {
+    const files =
+      ctx.request.files.file.length > 1
+        ? Array.from(ctx.request.files.file)
+        : [ctx.request.files.file]
+    usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await apiKeyTable.update(buildUpdateParams(ctx.apiKey, property, usage))
+    await usageQuota.update(ctx.apiKey, property, usage)
   } catch (err) {
-    if (err.code !== "ConditionalCheckFailedException") {
-      // get the API key so we can check it
-      let apiKey = await apiKeyTable.get({ primary: ctx.apiKey })
-      // we have infact breached the reset period
-      if (apiKey && apiKey.quotaReset >= Date.now()) {
-        // update the quota reset period and reset the values for all properties
-        apiKey.quotaReset = Date.now() + QUOTA_RESET
-        for (let prop of Object.keys(apiKey.usageQuota)) {
-          apiKey.usageQuota[prop] = 0
-        }
-        await apiKeyTable.put({ item: apiKey })
-      }
-      ctx.throw(403, `Resource limits have been reached`)
-    }
+    ctx.throw(403, err)
   }
-}
+}

packages/server/src/utilities/fileProcessor.js

@@ -1,5 +1,5 @@
 const fs = require("fs")
-const sharp = require("sharp")
+// const sharp = require("sharp")
 const fsPromises = fs.promises
 
 const FORMATS = {
@@ -7,14 +7,14 @@ const FORMATS = {
 }
 
 async function processImage(file) {
-  const imgMeta = await sharp(file.path)
-    .resize(300)
-    .toFile(file.outputPath)
-
-  return {
-    ...file,
-    ...imgMeta,
-  }
+  // const imgMeta = await sharp(file.path)
+  //   .resize(300)
+  //   .toFile(file.outputPath)
+  //
+  // return {
+  //   ...file,
+  //   ...imgMeta,
+  // }
 }
 
 async function process(file) {

packages/server/src/utilities/usageQuota.js

@@ -0,0 +1,72 @@
+const environment = require("../environment")
+const { apiKeyTable } = require("../db/dynamoClient")
+
+function buildUpdateParams(key, property, usage) {
+  return {
+    primary: key,
+    condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now",
+    expression: "ADD #quota.#prop :usage",
+    names: {
+      "#quota": "usageQuota",
+      "#prop": property,
+      "#limits": "limits",
+      "#quotaReset": "quotaReset",
+    },
+    values: {
+      ":usage": usage,
+      ":now": Date.now(),
+    },
+  }
+}
+
+// a normalised month in milliseconds
+const QUOTA_RESET = 2592000000
+
+exports.Properties = {
+  RECORD: "records",
+  UPLOAD: "storage",
+  VIEW: "views",
+  USER: "users",
+  AUTOMATION: "automationRuns",
+}
+
+exports.getAPIKey = async appId => {
+  return apiKeyTable.get({ primary: appId })
+}
+
+/**
+ * Given a specified API key this will add to the usage object for the specified property.
+ * @param {string} apiKey The API key which is to be updated.
+ * @param {string} property The property which is to be added to (within the nested usageQuota object).
+ * @param {number} usage The amount (this can be negative) to adjust the number by.
+ * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
+ * also been reset after this call.
+ */
+exports.update = async (apiKey, property, usage) => {
+  // don't try validate in builder
+  if (!environment.CLOUD) {
+    return
+  }
+  try {
+    await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
+  } catch (err) {
+    if (err.code !== "ConditionalCheckFailedException") {
+      // get the API key so we can check it
+      let apiKey = await apiKeyTable.get({ primary: apiKey })
+      // we have infact breached the reset period
+      if (apiKey && apiKey.quotaReset >= Date.now()) {
+        // update the quota reset period and reset the values for all properties
+        apiKey.quotaReset = Date.now() + QUOTA_RESET
+        for (let prop of Object.keys(apiKey.usageQuota)) {
+          if (prop === property) {
+            apiKey.usageQuota[prop] = usage > 0 ? usage : 0
+          } else {
+            apiKey.usageQuota[prop] = 0
+          }
+        }
+        await apiKeyTable.put({ item: apiKey })
+      }
+      throw "Resource limits have been reached"
+    }
+  }
+}