Adding start to redis caching.

packages/auth/src/redis/utils.js

@@ -16,6 +16,7 @@ exports.Databases = {
   USER_CACHE: "users",
   FLAGS: "flags",
   APP_METADATA: "appMetadata",
+  QUERY_VARS: "queryVars",
 }
 
 exports.SEPARATOR = SEPARATOR

packages/server/src/api/controllers/query/index.js

@@ -1,4 +1,3 @@
-const { processString } = require("@budibase/string-templates")
 const CouchDB = require("../../../db")
 const {
   generateQueryID,
@@ -90,47 +89,6 @@ exports.save = async function (ctx) {
   ctx.message = `Query ${query.name} saved successfully.`
 }
 
-async function enrichQueryFields(fields, parameters = {}) {
-  const enrichedQuery = {}
-
-  // enrich the fields with dynamic parameters
-  for (let key of Object.keys(fields)) {
-    if (fields[key] == null) {
-      continue
-    }
-    if (typeof fields[key] === "object") {
-      // enrich nested fields object
-      enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)
-    } else if (typeof fields[key] === "string") {
-      // enrich string value as normal
-      enrichedQuery[key] = await processString(fields[key], parameters, {
-        noHelpers: true,
-      })
-    } else {
-      enrichedQuery[key] = fields[key]
-    }
-  }
-
-  if (
-    enrichedQuery.json ||
-    enrichedQuery.customData ||
-    enrichedQuery.requestBody
-  ) {
-    try {
-      enrichedQuery.json = JSON.parse(
-        enrichedQuery.json ||
-          enrichedQuery.customData ||
-          enrichedQuery.requestBody
-      )
-    } catch (err) {
-      // no json found, ignore
-    }
-    delete enrichedQuery.customData
-  }
-
-  return enrichedQuery
-}
-
 exports.find = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const query = enrichQueries(await db.get(ctx.params.queryId))
@@ -148,13 +106,13 @@ exports.preview = async function (ctx) {
   const datasource = await db.get(ctx.request.body.datasourceId)
 
   const { fields, parameters, queryVerb, transformer } = ctx.request.body
-  const enrichedQuery = await enrichQueryFields(fields, parameters)
 
   try {
     const { rows, keys, info, extra } = await Runner.run({
       datasource,
       queryVerb,
-      query: enrichedQuery,
+      fields,
+      parameters,
       transformer,
     })
 
@@ -175,17 +133,13 @@ async function execute(ctx, opts = { rowsOnly: false }) {
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
 
-  const enrichedQuery = await enrichQueryFields(
-    query.fields,
-    ctx.request.body.parameters
-  )
-
   // call the relevant CRUD method on the integration class
   try {
     const { rows, extra } = await Runner.run({
       datasource,
       queryVerb: query.queryVerb,
-      query: enrichedQuery,
+      fields: query.fields,
+      parameters: ctx.request.body.parameter,
       transformer: query.transformer,
     })
     if (opts && opts.rowsOnly) {

packages/server/src/threads/query.js

@@ -1,6 +1,66 @@
 require("./utils").threadSetup()
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
+const { processStringSync } = require("@budibase/string-templates")
+
+async function addDatasourceVariables(datasource, parameters) {
+  if (!datasource || !datasource.config) {
+    return parameters
+  }
+  const staticVars = datasource.config.staticVariables || {}
+  const dynamicVars = datasource.config.dynamicVariables || []
+  for (let [key, value] of Object.entries(staticVars)) {
+    if (!parameters[key]) {
+      parameters[key] = value
+    }
+  }
+  for (let variable of dynamicVars) {
+    console.log(variable)
+    // TODO: get the variable from query
+  }
+  return parameters
+}
+
+function enrichQueryFields(fields, parameters = {}) {
+  const enrichedQuery = {}
+
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = enrichQueryFields(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noHelpers: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+
+  return enrichedQuery
+}
 
 function formatResponse(resp) {
   if (typeof resp === "string") {
@@ -22,7 +82,16 @@ function hasExtraData(response) {
   )
 }
 
-async function runAndTransform(datasource, queryVerb, query, transformer) {
+async function runAndTransform(
+  datasource,
+  queryVerb,
+  fields,
+  parameters,
+  transformer
+) {
+  // pre-query, make sure datasource variables are added to parameters
+  parameters = await addDatasourceVariables(datasource, parameters)
+  const query = enrichQueryFields(fields, parameters)
   const Integration = integrations[datasource.source]
   if (!Integration) {
     throw "Integration type does not exist."
@@ -69,7 +138,8 @@ module.exports = (input, callback) => {
   runAndTransform(
     input.datasource,
     input.queryVerb,
-    input.query,
+    input.fields,
+    input.parameters,
     input.transformer
   )
     .then(response => {

packages/server/src/threads/utils.js

@@ -1,6 +1,18 @@
 const env = require("../environment")
 const CouchDB = require("../db")
 const { init } = require("@budibase/auth")
+const redis = require("@budibase/auth/redis")
+const { SEPARATOR } = require("@budibase/auth/db")
+
+let client
+
+if (!client) {
+  client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
+}
+
+process.on("exit", async () => {
+  if (client) await client.finish()
+})
 
 exports.threadSetup = () => {
   // don't run this if not threading
@@ -11,3 +23,11 @@ exports.threadSetup = () => {
   env.setInThread()
   init(CouchDB)
 }
+
+function makeVariableKey(queryId, variable) {
+  return `${queryId}${SEPARATOR}${variable}`
+}
+
+exports.checkCacheForDynamicVariable = async (queryId, variable) => {
+  await client.get(makeVariableKey(queryId, variable))
+}