Updates for API usage after testing against local Dynamo.

packages/server/scripts/createApiKeyAndAppId.js

@@ -0,0 +1,55 @@
+// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running
+const dynamoClient = require("../src/db/dynamoClient")
+
+if (process.argv[2] == null || process.argv[3] == null) {
+  console.error(
+    "Inputs incorrect format, was expecting: node createApiKeyAndAppId.js <API_KEY> <APP_ID>"
+  )
+  process.exit(-1)
+}
+
+const FAKE_STRING = "fakestring"
+
+// set fake credentials for local dynamo to actually work
+process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
+process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
+dynamoClient.init("http://localhost:8333")
+
+async function run() {
+  await dynamoClient.apiKeyTable.put({
+    item: {
+      pk: process.argv[2],
+      accountId: FAKE_STRING,
+      trackingId: FAKE_STRING,
+      quotaReset: Date.now() + 2592000000,
+      usageQuota: {
+        automationRuns: 0,
+        records: 0,
+        storage: 0,
+        users: 0,
+        views: 0,
+      },
+      usageLimits: {
+        automationRuns: 10,
+        records: 10,
+        storage: 1000,
+        users: 10,
+        views: 10,
+      },
+    },
+  })
+  await dynamoClient.apiKeyTable.put({
+    item: {
+      pk: process.argv[3],
+      apiKey: process.argv[2],
+    },
+  })
+}
+
+run()
+  .then(() => {
+    console.log("Records should have been created.")
+  })
+  .catch(err => {
+    console.error("Cannot create records - " + err)
+  })

packages/server/src/automations/index.js

@@ -21,9 +21,10 @@ function runWorker(job) {
 
 async function updateQuota(automation) {
   const appId = automation.appId
-  const apiKey = await getAPIKey(appId)
+  const apiObj = await getAPIKey(appId)
   // this will fail, causing automation to escape if limits reached
-  await update(apiKey, Properties.AUTOMATION, 1)
+  await update(apiObj.apiKey, Properties.AUTOMATION, 1)
+  return apiObj.apiKey
 }
 
 /**
@@ -33,8 +34,8 @@ module.exports.init = function() {
   actions.init().then(() => {
     triggers.automationQueue.process(async job => {
       try {
-        if (environment.CLOUD) {
+        if (environment.CLOUD && job.data.automation) {
-          await updateQuota(job.data.automation)
+          job.data.automation.apiKey = await updateQuota(job.data.automation)
         }
         if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
           await runWorker(job)

packages/server/src/automations/steps/createRecord.js

@@ -1,5 +1,7 @@
 const recordController = require("../../api/controllers/record")
 const automationUtils = require("../automationUtils")
+const environment = require("../../environment")
+const usage = require("../../utilities/usageQuota")
 
 module.exports.definition = {
   name: "Create Row",
@@ -56,7 +58,7 @@ module.exports.definition = {
   },
 }
 
-module.exports.run = async function({ inputs, instanceId }) {
+module.exports.run = async function({ inputs, instanceId, apiKey }) {
   // TODO: better logging of when actions are missed due to missing parameters
   if (inputs.record == null || inputs.record.modelId == null) {
     return
@@ -78,6 +80,9 @@ module.exports.run = async function({ inputs, instanceId }) {
   }
 
   try {
+    if (environment.CLOUD) {
+      await usage.update(apiKey, usage.Properties.RECORD, 1)
+    }
     await recordController.save(ctx)
     return {
       record: inputs.record,

packages/server/src/automations/steps/createUser.js

@@ -1,5 +1,7 @@
 const accessLevels = require("../../utilities/accessLevels")
 const userController = require("../../api/controllers/user")
+const environment = require("../../environment")
+const usage = require("../../utilities/usageQuota")
 
 module.exports.definition = {
   description: "Create a new user",
@@ -56,7 +58,7 @@ module.exports.definition = {
   },
 }
 
-module.exports.run = async function({ inputs, instanceId }) {
+module.exports.run = async function({ inputs, instanceId, apiKey }) {
   const { username, password, accessLevelId } = inputs
   const ctx = {
     user: {
@@ -68,6 +70,9 @@ module.exports.run = async function({ inputs, instanceId }) {
   }
 
   try {
+    if (environment.CLOUD) {
+      await usage.update(apiKey, usage.Properties.USER, 1)
+    }
     await userController.create(ctx)
     return {
       response: ctx.body,

packages/server/src/automations/steps/deleteRecord.js

@@ -1,4 +1,6 @@
 const recordController = require("../../api/controllers/record")
+const environment = require("../../environment")
+const usage = require("../../utilities/usageQuota")
 
 module.exports.definition = {
   description: "Delete a row from your database",
@@ -48,7 +50,7 @@ module.exports.definition = {
   },
 }
 
-module.exports.run = async function({ inputs, instanceId }) {
+module.exports.run = async function({ inputs, instanceId, apiKey }) {
   // TODO: better logging of when actions are missed due to missing parameters
   if (inputs.id == null || inputs.revision == null) {
     return
@@ -63,6 +65,9 @@ module.exports.run = async function({ inputs, instanceId }) {
   }
 
   try {
+    if (environment.CLOUD) {
+      await usage.update(apiKey, usage.Properties.RECORD, -1)
+    }
     await recordController.destroy(ctx)
     return {
       response: ctx.body,

packages/server/src/automations/thread.js

@@ -62,6 +62,7 @@ class Orchestrator {
         const outputs = await stepFn({
           inputs: step.inputs,
           instanceId: this._instanceId,
+          apiKey: automation.apiKey,
         })
         if (step.stepId === FILTER_STEP_ID && !outputs.success) {
           break

packages/server/src/db/dynamoClient.js

@@ -1,6 +1,8 @@
 let _ = require("lodash")
 let environment = require("../environment")
 
+const AWS_REGION = environment.AWS_REGION ? environment.AWS_REGION : "eu-west-1"
+
 const TableInfo = {
   API_KEYS: {
     name: "beta-api-key-table",
@@ -49,6 +51,7 @@ class Table {
     condition,
     names,
     values,
+    exists,
     otherProps,
   }) {
     let params = {
@@ -66,6 +69,13 @@ class Table {
     if (this._sort && sort) {
       params.Key[this._sort] = sort
     }
+    if (exists) {
+      params.ExpressionAttributeNames["#PRIMARY"] = this._primary
+      if (params.ConditionExpression) {
+        params.ConditionExpression += " AND "
+      }
+      params.ConditionExpression += "attribute_exists(#PRIMARY)"
+    }
     if (otherProps) {
       params = _.merge(params, otherProps)
     }
@@ -90,15 +100,17 @@ class Table {
   }
 }
 
-exports.init = () => {
+exports.init = endpoint => {
-  if (!environment.CLOUD) {
-    return
-  }
   let AWS = require("aws-sdk")
+  AWS.config.update({
+    region: AWS_REGION,
+  })
   let docClientParams = {
     correctClockSkew: true,
   }
-  if (environment.DYNAMO_ENDPOINT) {
+  if (endpoint) {
+    docClientParams.endpoint = endpoint
+  } else if (environment.DYNAMO_ENDPOINT) {
     docClientParams.endpoint = environment.DYNAMO_ENDPOINT
   }
   docClient = new AWS.DynamoDB.DocumentClient(docClientParams)
@@ -106,3 +118,11 @@ exports.init = () => {
 
 exports.apiKeyTable = new Table(TableInfo.API_KEYS)
 exports.userTable = new Table(TableInfo.USERS)
+
+if (environment.CLOUD) {
+  exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`)
+} else {
+  process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
+  process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
+  exports.init("http://localhost:8333")
+}

packages/server/src/middleware/authorized.js

@@ -16,8 +16,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
   }
 
   if (ctx.user.accessLevel._id === BUILDER_LEVEL_ID) {
-    await next()
+    return next()
-    return
   }
 
   if (permName === BUILDER) {
@@ -28,8 +27,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
   const permissionId = ({ name, itemId }) => name + (itemId ? `-${itemId}` : "")
 
   if (ctx.user.accessLevel._id === ADMIN_LEVEL_ID) {
-    await next()
+    return next()
-    return
   }
 
   const thisPermissionId = permissionId({
@@ -42,8 +40,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
     ctx.user.accessLevel._id === POWERUSER_LEVEL_ID &&
     !adminPermissions.map(permissionId).includes(thisPermissionId)
   ) {
-    await next()
+    return next()
-    return
   }
 
   if (
@@ -51,8 +48,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
       .map(permissionId)
       .includes(thisPermissionId)
   ) {
-    await next()
+    return next()
-    return
   }
 
   ctx.throw(403, "Not Authorized")

packages/server/src/middleware/usageQuota.js

@@ -1,5 +1,6 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
+const environment = require("../environment")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -29,11 +30,9 @@ module.exports = async (ctx, next) => {
   const db = new CouchDB(ctx.user.instanceId)
   let usage = METHOD_MAP[ctx.req.method]
   const property = getProperty(ctx.req.url)
-  console.log(ctx.req.url)
   if (usage == null || property == null) {
     return next()
   }
-  console.log(`${usage} to ${property}`)
   // post request could be a save of a pre-existing entry
   if (ctx.request.body && ctx.request.body._id) {
     try {
@@ -52,8 +51,12 @@ module.exports = async (ctx, next) => {
         : [ctx.request.files.file]
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
+  if (!environment.CLOUD) {
+    return next()
+  }
   try {
     await usageQuota.update(ctx.apiKey, property, usage)
+    return next()
   } catch (err) {
     ctx.throw(403, err)
   }

packages/server/src/utilities/builder/setBuilderToken.js

@@ -8,7 +8,9 @@ module.exports = (ctx, appId, instanceId) => {
     instanceId,
     appId,
   }
-
+  if (process.env.BUDIBASE_API_KEY) {
+    builderUser.apiKey = process.env.BUDIBASE_API_KEY
+  }
   const token = jwt.sign(builderUser, ctx.config.jwtSecret, {
     expiresIn: "30 days",
   })

packages/server/src/utilities/usageQuota.js

@@ -4,12 +4,12 @@ const { apiKeyTable } = require("../db/dynamoClient")
 function buildUpdateParams(key, property, usage) {
   return {
     primary: key,
-    condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now",
+    condition: "#quota.#prop < #limits.#prop AND #quotaReset > :now",
     expression: "ADD #quota.#prop :usage",
     names: {
       "#quota": "usageQuota",
       "#prop": property,
-      "#limits": "limits",
+      "#limits": "usageLimits",
       "#quotaReset": "quotaReset",
     },
     values: {
@@ -50,21 +50,22 @@ exports.update = async (apiKey, property, usage) => {
   try {
     await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
   } catch (err) {
-    if (err.code !== "ConditionalCheckFailedException") {
+    if (err.code === "ConditionalCheckFailedException") {
       // get the API key so we can check it
-      let apiKey = await apiKeyTable.get({ primary: apiKey })
+      const keyObj = await apiKeyTable.get({ primary: apiKey })
       // we have infact breached the reset period
-      if (apiKey && apiKey.quotaReset >= Date.now()) {
+      if (keyObj && keyObj.quotaReset <= Date.now()) {
         // update the quota reset period and reset the values for all properties
-        apiKey.quotaReset = Date.now() + QUOTA_RESET
+        keyObj.quotaReset = Date.now() + QUOTA_RESET
-        for (let prop of Object.keys(apiKey.usageQuota)) {
+        for (let prop of Object.keys(keyObj.usageQuota)) {
           if (prop === property) {
-            apiKey.usageQuota[prop] = usage > 0 ? usage : 0
+            keyObj.usageQuota[prop] = usage > 0 ? usage : 0
           } else {
-            apiKey.usageQuota[prop] = 0
+            keyObj.usageQuota[prop] = 0
           }
         }
-        await apiKeyTable.put({ item: apiKey })
+        await apiKeyTable.put({ item: keyObj })
+        return
       }
       throw "Resource limits have been reached"
     }