Merge branch 'feature/query-variables' of github.com:Budibase/budibase into rest-pagination

5 years ago · 932a63de7f
13 changed files with 114 additions and 28 deletions
--- a/packages/auth/src/constants.js
+++ b/packages/auth/src/constants.js
@ -16,6 +16,7 @@ exports.Headers = {
  APP_ID: "x-budibase-app-id",
  TYPE: "x-budibase-type",
  TENANT_ID: "x-budibase-tenant-id",
+  TOKEN: "x-budibase-token",
 }

 exports.GlobalRoles = {
--- a/packages/auth/src/middleware/authenticated.js
+++ b/packages/auth/src/middleware/authenticated.js
@ -1,5 +1,5 @@
 const { Cookies, Headers } = require("../constants")
-const { getCookie, clearCookie } = require("../utils")
+const { getCookie, clearCookie, openJwt } = require("../utils")
 const { getUser } = require("../cache/user")
 const { getSession, updateSessionTTL } = require("../security/sessions")
 const { buildMatcherRegex, matches } = require("./matchers")
@ -35,8 +35,9 @@ module.exports = (
      publicEndpoint = true
    }
    try {
-      // check the actual user is authenticated first
-      const authCookie = getCookie(ctx, Cookies.Auth)
+      // check the actual user is authenticated first, try header or cookie
+      const headerToken = ctx.request.headers[Headers.TOKEN]
+      const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken)
      let authenticated = false,
        user = null,
        internal = false
--- a/packages/auth/src/utils.js
+++ b/packages/auth/src/utils.js
@ -63,6 +63,17 @@ exports.getAppId = ctx => {
  return appId
 }

+/**
+ * opens the contents of the specified encrypted JWT.
+ * @return {object} the contents of the token.
+ */
+exports.openJwt = token => {
+  if (!token) {
+    return token
+  }
+  return jwt.verify(token, options.secretOrKey)
+}
+
 /**
 * Get a cookie from context, and decrypt if necessary.
 * @param {object} ctx The request which is to be manipulated.
@ -75,7 +86,7 @@ exports.getCookie = (ctx, name) => {
    return cookie
  }

-  return jwt.verify(cookie, options.secretOrKey)
+  return exports.openJwt(cookie)
 }

 /**
--- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte
@ -60,9 +60,9 @@
  </div>
 </div>
 <Body size="S"
-  >Variables enabled you to store and reuse values in queries. Static variables
-  use constant values while dynamic values can be bound to the response headers
-  or body of a query</Body
+  >Variables enable you to store and re-use values in queries, with the choice
+  of a static value such as a token using static variables, or a value from a
+  query response using dynamic variables.</Body
 >
 <Heading size="XS">Static</Heading>
 <Layout noPadding gap="XS">
@ -78,7 +78,7 @@
 <Heading size="XS">Dynamic</Heading>
 <Body size="S">
  Dynamic variables are evaluated when a dependant query is executed. The value
-  is cached for 24 hours and will re-evaluate if the dependendent query fails.
+  is cached for a period of time and will be refreshed if a query fails.
 </Body>
 <ViewDynamicVariables {queries} {datasource} />

--- a/packages/builder/src/helpers/data/utils.js
+++ b/packages/builder/src/helpers/data/utils.js
@ -121,10 +121,13 @@ export function flipHeaderState(headersActivity) {
 }

 // convert dynamic variables list to simple key/val object
-export function variablesToObject(datasource) {
+export function getDynamicVariables(datasource, queryId) {
  const variablesList = datasource?.config?.dynamicVariables
  if (variablesList && variablesList.length > 0) {
-    return variablesList.reduce(
+    const filtered = queryId
+      ? variablesList.filter(variable => variable.queryId === queryId)
+      : variablesList
+    return filtered.reduce(
      (acc, next) => ({ ...acc, [next.name]: next.value }),
      {}
    )
@ -133,10 +136,10 @@ export function variablesToObject(datasource) {
 }

 // convert dynamic variables object back to a list, enrich with query id
-export function rebuildVariables(queryId, variables) {
-  let vars = []
+export function rebuildVariables(datasource, queryId, variables) {
+  let newVariables = []
  if (variables) {
-    vars = Object.entries(variables).map(entry => {
+    newVariables = Object.entries(variables).map(entry => {
      return {
        name: entry[0],
        value: entry[1],
@ -144,7 +147,15 @@ export function rebuildVariables(queryId, variables) {
      }
    })
  }
-  return vars
+  let existing = datasource?.config?.dynamicVariables || []
+  // filter out any by same name
+  existing = existing.filter(
+    variable =>
+      !newVariables.find(
+        newVar => newVar.name.toLowerCase() === variable.name.toLowerCase()
+      )
+  )
+  return [...existing, ...newVariables]
 }

 export function shouldShowVariables(dynamicVariables, variablesReadOnly) {
@ -173,7 +184,7 @@ export default {
  keyValueToQueryParameters,
  queryParametersToKeyValue,
  schemaToFields,
-  variablesToObject,
+  getDynamicVariables,
  rebuildVariables,
  shouldShowVariables,
  buildAuthConfigs,
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte
@ -1,10 +1,11 @@
 <script>
-  import { Input, ModalContent, Modal } from "@budibase/bbui"
+  import { Input, ModalContent, Modal, Body } from "@budibase/bbui"

  export let dynamicVariables
+  export let datasource
  export let binding

-  let name, modal, valid
+  let name, modal, valid, allVariableNames

  export const show = () => {
    modal.show()
@ -17,11 +18,15 @@
    if (!name) {
      return false
    }
-    const varKeys = Object.keys(vars || {})
-    return varKeys.find(key => key.toLowerCase() === name.toLowerCase()) == null
+    return !allVariableNames.find(
+      varName => varName.toLowerCase() === name.toLowerCase()
+    )
  }

  $: valid = checkValid(dynamicVariables, name)
+  $: allVariableNames = (datasource?.config?.dynamicVariables || []).map(
+    variable => variable.name
+  )
  $: error = name && !valid ? "Variable name is already in use." : null

  async function saveVariable() {
@ -40,6 +45,10 @@
    onConfirm={saveVariable}
    disabled={!valid}
  >
+    <Body size="S"
+      >Specify a name for your new dynamic variable, this must be unique across
+      your datasource.</Body
+    >
    <Input label="Variable name" bind:value={name} on:input {error} />
  </ModalContent>
 </Modal>
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
@ -118,6 +118,7 @@

      if (dynamicVariables) {
        datasource.config.dynamicVariables = restUtils.rebuildVariables(
+          datasource,
          saveId,
          dynamicVariables
        )
@ -218,7 +219,7 @@
    if (query && !query.fields.pagination) {
      query.fields.pagination = {}
    }
-    dynamicVariables = restUtils.variablesToObject(datasource)
+    dynamicVariables = restUtils.getDynamicVariables(datasource, query._id)
  })
 </script>

@ -438,9 +439,10 @@
            {#if showVariablesTab}
              <Tab title="Dynamic Variables">
                <Layout noPadding gap="S">
-                  <Body size="S"
-                    >{"Create dynamic variables to use body and headers results in other queries"}</Body
-                  >
+                  <Body size="S">
+                    Create dynamic variables based on response body or headers
+                    from other queries.
+                  </Body>
                  <KeyValueBuilder
                    bind:object={dynamicVariables}
                    name="Variable"
--- a/packages/builder/src/stores/backend/queries.js
+++ b/packages/builder/src/stores/backend/queries.js
@ -91,6 +91,7 @@ export function createQueriesStore() {
          {}
        ),
        datasourceId: query.datasourceId,
+        queryId: query._id || undefined,
      })

      if (response.status !== 200) {
--- a/packages/server/src/api/controllers/query/index.js
+++ b/packages/server/src/api/controllers/query/index.js
@ -104,8 +104,10 @@ exports.preview = async function (ctx) {
  const db = new CouchDB(ctx.appId)

  const datasource = await db.get(ctx.request.body.datasourceId)
-
-  const { fields, parameters, queryVerb, transformer } = ctx.request.body
+  // preview may not have a queryId as it hasn't been saved, but if it does
+  // this stops dynamic variables from calling the same query
+  const { fields, parameters, queryVerb, transformer, queryId } =
+    ctx.request.body

  try {
    const { rows, keys, info, extra } = await Runner.run({
@ -115,6 +117,7 @@ exports.preview = async function (ctx) {
      fields,
      parameters,
      transformer,
+      queryId,
    })

    ctx.body = {
@ -143,6 +146,7 @@ async function execute(ctx, opts = { rowsOnly: false }) {
      fields: query.fields,
      parameters: ctx.request.body.parameter,
      transformer: query.transformer,
+      queryId: ctx.params.queryId,
    })
    if (opts && opts.rowsOnly) {
      ctx.body = rows
--- a/packages/server/src/api/controllers/query/validation.js
+++ b/packages/server/src/api/controllers/query/validation.js
@ -37,6 +37,7 @@ exports.generateQueryPreviewValidation = () => {
    extra: Joi.object().optional(),
    datasourceId: Joi.string().required(),
    transformer: Joi.string().optional(),
-    parameters: Joi.object({}).required().unknown(true)
+    parameters: Joi.object({}).required().unknown(true),
+    queryId: Joi.string().optional(),
  }))
 }
--- a/packages/server/src/threads/query.js
+++ b/packages/server/src/threads/query.js
@ -13,7 +13,14 @@ class QueryRunner {
    this.fields = input.fields
    this.parameters = input.parameters
    this.transformer = input.transformer
+    this.queryId = input.queryId
    this.noRecursiveQuery = flags.noRecursiveQuery
+    this.cachedVariables = []
+    // allows the response from a query to be stored throughout this
+    // execution so that if it needs to be re-used for another variable
+    // it can be
+    this.queryResponse = {}
+    this.hasRerun = false
  }

  async execute() {
@ -43,6 +50,19 @@ class QueryRunner {
      rows = runner.execute()
    }

+    // if the request fails we retry once, invalidating the cached value
+    if (
+      info &&
+      info.code >= 400 &&
+      this.cachedVariables.length > 0 &&
+      !this.hasRerun
+    ) {
+      this.hasRerun = true
+      // invalidate the cache value
+      await threadUtils.invalidateDynamicVariables(this.cachedVariables)
+      return this.execute()
+    }
+
    // needs to an array for next step
    if (!Array.isArray(rows)) {
      rows = [rows]
@ -86,8 +106,14 @@ class QueryRunner {
      name = variable.name
    let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
    if (!value) {
-      value = await this.runAnotherQuery(queryId, parameters)
+      value = this.queryResponse[queryId]
+        ? this.queryResponse[queryId]
+        : await this.runAnotherQuery(queryId, parameters)
+      // store incase this query is to be called again
+      this.queryResponse[queryId] = value
      await threadUtils.storeDynamicVariable(queryId, name, value)
+    } else {
+      this.cachedVariables.push({ queryId, name })
    }
    return value
  }
@ -108,6 +134,10 @@ class QueryRunner {
      // need to see if this uses any variables
      const stringFields = JSON.stringify(fields)
      const foundVars = dynamicVars.filter(variable => {
+        // don't allow a query to use its own dynamic variable (loop)
+        if (variable.queryId === this.queryId) {
+          return false
+        }
        // look for {{ variable }} but allow spaces between handlebars
        const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
        return regex.test(stringFields)
@ -120,6 +150,8 @@ class QueryRunner {
          data: responses[i].rows,
          info: responses[i].extra,
        })
+        // make sure its known that this uses dynamic variables in case it fails
+        this.hasDynamicVariables = true
      }
    }
    return parameters
--- a/packages/server/src/threads/utils.js
+++ b/packages/server/src/threads/utils.js
@ -38,6 +38,16 @@ exports.checkCacheForDynamicVariable = async (queryId, variable) => {
  return cache.get(makeVariableKey(queryId, variable))
 }

+exports.invalidateDynamicVariables = async cachedVars => {
+  let promises = []
+  for (let variable of cachedVars) {
+    promises.push(
+      client.delete(makeVariableKey(variable.queryId, variable.name))
+    )
+  }
+  await Promise.all(promises)
+}
+
 exports.storeDynamicVariable = async (queryId, variable, value) => {
  const cache = await getClient()
  await cache.store(
--- a/packages/worker/src/api/controllers/global/auth.js
+++ b/packages/worker/src/api/controllers/global/auth.js
@ -12,7 +12,7 @@ const {
  hash,
  platformLogout,
 } = authPkg.utils
-const { Cookies } = authPkg.constants
+const { Cookies, Headers } = authPkg.constants
 const { passport } = authPkg.auth
 const { checkResetPasswordCode } = require("../../../utilities/redis")
 const {
@ -60,7 +60,10 @@ async function authInternal(ctx, user, err = null, info = null) {
    return ctx.throw(403, info ? info : "Unauthorized")
  }

+  // set a cookie for browser access
  setCookie(ctx, user.token, Cookies.Auth, { sign: false })
+  // set the token in a header as well for APIs
+  ctx.set(Headers.TOKEN, user.token)
  // get rid of any app cookies on login
  // have to check test because this breaks cypress
  if (!env.isTest()) {