Adding the ability to filter using the filter drawer for formula fields, getting them based on the enriched contents of a row - adding functionality to denote to each table which other tables depend on them for formula.

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte

@@ -131,7 +131,7 @@
                 {bindings}
                 on:change={event => (filter.value = event.detail)}
               />
-            {:else if ["string", "longform", "number"].includes(filter.type)}
+            {:else if ["string", "longform", "number", "formula"].includes(filter.type)}
               <Input disabled={filter.noValue} bind:value={filter.value} />
             {:else if ["options", "array"].includes(filter.type)}
               <Combobox

packages/builder/src/constants/lucene.js

@@ -59,24 +59,26 @@ export const NoEmptyFilterStrings = [
  */
 export const getValidOperatorsForType = type => {
   const Op = OperatorOptions
+  const stringOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.StartsWith,
+    Op.Like,
+    Op.Empty,
+    Op.NotEmpty,
+  ]
+  const numOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.MoreThan,
+    Op.LessThan,
+    Op.Empty,
+    Op.NotEmpty,
+  ]
   if (type === "string") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.StartsWith,
-      Op.Like,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
+    return stringOps
   } else if (type === "number") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.MoreThan,
-      Op.LessThan,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
+    return numOps
   } else if (type === "options") {
     return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
   } else if (type === "array") {
@@ -84,23 +86,11 @@ export const getValidOperatorsForType = type => {
   } else if (type === "boolean") {
     return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
   } else if (type === "longform") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.StartsWith,
-      Op.Like,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
+    return stringOps
   } else if (type === "datetime") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.MoreThan,
-      Op.LessThan,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
+    return numOps
+  } else if (type === "formula") {
+    return stringOps.concat([Op.MoreThan, Op.LessThan])
   }
   return []
 }

packages/builder/src/helpers/searchFields.js

@@ -27,5 +27,8 @@ export function getFields(fields, { allowLinks } = { allowLinks: true }) {
       filteredFields = filteredFields.concat(getTableFields(linkField))
     }
   }
-  return filteredFields
+  const staticFormulaFields = fields.filter(
+    field => field.type === "formula" && field.formulaType === "static"
+  )
+  return filteredFields.concat(staticFormulaFields)
 }

packages/server/src/api/controllers/row/internal.js

@@ -12,6 +12,7 @@ const {
   outputProcessing,
   processAutoColumn,
   cleanupAttachments,
+  processFormulas,
 } = require("../../../utilities/rowProcessor")
 const { FieldTypes } = require("../../../constants")
 const { isEqual } = require("lodash")
@@ -36,6 +37,17 @@ const CALCULATION_TYPES = {
 
 async function storeResponse(ctx, db, row, oldTable, table) {
   row.type = "row"
+  let rowToSave = cloneDeep(row)
+  // process the row before return, to include relationships
+  let enrichedRow = await outputProcessing(ctx, table, cloneDeep(row), {
+    squash: false,
+  })
+  // use enriched row to generate formulas for saving, specifically only use as context
+  row = processFormulas(table, row, {
+    dynamic: false,
+    contextRows: enrichedRow,
+  })
+
   // don't worry about rev, tables handle rev/lastID updates
   // if another row has been written since processing this will
   // handle the auto ID clash
@@ -45,7 +57,7 @@ async function storeResponse(ctx, db, row, oldTable, table) {
     } catch (err) {
       if (err.status === 409) {
         const updatedTable = await db.get(table._id)
-        let response = processAutoColumn(null, updatedTable, row, {
+        let response = processAutoColumn(null, updatedTable, rowToSave, {
           reprocessing: true,
         })
         await db.put(response.table)
@@ -56,10 +68,10 @@ async function storeResponse(ctx, db, row, oldTable, table) {
     }
   }
   const response = await db.put(row)
-  row._rev = response.rev
-  // process the row before return, to include relationships
-  row = await outputProcessing(ctx, table, row, { squash: false })
-  return { row, table }
+  // for response, calculate the formulas for the enriched row
+  enrichedRow._rev = response.rev
+  enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
+  return { row: enrichedRow, table }
 }
 
 // doesn't do the outputProcessing

packages/server/src/api/controllers/table/index.js

@@ -3,12 +3,8 @@ const internal = require("./internal")
 const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
 const { isExternalTable, isSQL } = require("../../../integrations/utils")
-const {
-  getTableParams,
-  getDatasourceParams,
-  BudibaseInternalDB,
-} = require("../../../db/utils")
-const { getTable } = require("./utils")
+const { getDatasourceParams } = require("../../../db/utils")
+const { getTable, getAllInternalTables } = require("./utils")
 
 function pickApi({ tableId, table }) {
   if (table && !tableId) {
@@ -26,17 +22,7 @@ function pickApi({ tableId, table }) {
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-
-  const internal = internalTables.rows.map(tableDoc => ({
-    ...tableDoc.doc,
-    type: "internal",
-    sourceId: BudibaseInternalDB._id,
-  }))
+  const internal = await getAllInternalTables({ db })
 
   const externalTables = await db.allDocs(
     getDatasourceParams("plus", {

packages/server/src/api/controllers/table/internal.js

@@ -1,14 +1,82 @@
 const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const { getRowParams, generateTableID } = require("../../../db/utils")
-const { FieldTypes } = require("../../../constants")
+const { FieldTypes, FormulaTypes } = require("../../../constants")
 const {
   TableSaveFunctions,
   hasTypeChanged,
   getTable,
   handleDataImport,
+  getAllInternalTables,
 } = require("./utils")
 const usageQuota = require("../../../utilities/usageQuota")
+const { doesContainString } = require("@budibase/string-templates")
+const { cloneDeep } = require("lodash/fp")
+const { isEqual } = require("lodash")
+
+/**
+ * This function adds a note to related tables that they are
+ * used in a static formula - so that the link controller
+ * can manage hydrating related rows formula fields. This is
+ * specifically only for static formula.
+ */
+async function updateRelatedTablesForFormula(db, table) {
+  // start by retrieving all tables, remove the current table from the list
+  const tables = (await getAllInternalTables({ db })).filter(
+    tbl => tbl._id !== table._id
+  )
+  // clone the tables, so we can compare at end
+  const initialTables = cloneDeep(tables)
+  // first find the related column names
+  const relatedColumns = Object.values(table.schema).filter(
+    col => col.type === FieldTypes.LINK
+  )
+  // we start by removing the formula field from all tables
+  for (let otherTable of tables) {
+    if (!otherTable.relatedFormula) {
+      continue
+    }
+    const index = otherTable.relatedFormula.indexOf(table._id)
+    if (index !== -1) {
+      otherTable.relatedFormula.splice(index, 1)
+    }
+  }
+  for (let column of Object.values(table.schema)) {
+    // not a static formula, or doesn't contain a relationship
+    if (
+      column.type !== FieldTypes.FORMULA ||
+      column.formulaType !== FormulaTypes.STATIC
+    ) {
+      continue
+    }
+    // check to see if any relationship columns are used in formula
+    for (let relatedCol of relatedColumns) {
+      if (!doesContainString(column.formula, relatedCol.name)) {
+        continue
+      }
+      const relatedTable = tables.find(
+        related => related._id === relatedCol.tableId
+      )
+      // check if the table is already in the list of related formula, if it isn't, then add it
+      if (
+        relatedTable &&
+        (!relatedTable.relatedFormula ||
+          relatedTable.relatedFormula.indexOf(table._id) === -1)
+      ) {
+        relatedTable.relatedFormula = relatedTable.relatedFormula
+          ? [...relatedTable.relatedFormula, table._id]
+          : [table._id]
+      }
+    }
+  }
+  // now we just need to compare all the tables and see if any need saved
+  for (let initial of initialTables) {
+    const found = tables.find(tbl => initial._id === tbl._id)
+    if (found && !isEqual(initial, found)) {
+      await db.put(found)
+    }
+  }
+}
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
@@ -104,6 +172,8 @@ exports.save = async function (ctx) {
   tableToSave._rev = result.rev
 
   tableToSave = await tableSaveFunctions.after(tableToSave)
+  // has to run after, make sure it has _id
+  await updateRelatedTablesForFormula(db, tableToSave)
 
   return tableToSave
 }

packages/server/src/api/controllers/table/utils.js

@@ -4,6 +4,8 @@ const {
   getRowParams,
   generateRowID,
   InternalTables,
+  getTableParams,
+  BudibaseInternalDB,
 } = require("../../../db/utils")
 const { isEqual } = require("lodash/fp")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
@@ -230,8 +232,26 @@ class TableSaveFunctions {
   }
 }
 
-exports.getAllExternalTables = async (appId, datasourceId) => {
-  const db = new CouchDB(appId)
+exports.getAllInternalTables = async ({ db, appId }) => {
+  if (appId && !db) {
+    db = new CouchDB(appId)
+  }
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return internalTables.rows.map(tableDoc => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
+}
+
+exports.getAllExternalTables = async ({ db, appId }, datasourceId) => {
+  if (appId && !db) {
+    db = new CouchDB(appId)
+  }
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
@@ -240,7 +260,7 @@ exports.getAllExternalTables = async (appId, datasourceId) => {
 }
 
 exports.getExternalTable = async (appId, datasourceId, tableName) => {
-  const entities = await exports.getAllExternalTables(appId, datasourceId)
+  const entities = await exports.getAllExternalTables({ appId }, datasourceId)
   return entities[tableName]
 }
 

packages/server/src/db/linkedRows/LinkController.js

@@ -156,6 +156,8 @@ class LinkController {
     return true
   }
 
+  updateRelatedFormula() {}
+
   /**
    * Given the link field of this table, and the link field of the linked table, this makes sure
    * the state of relationship type is accurate on both.

packages/server/src/definitions/common.ts

@@ -17,6 +17,8 @@ export interface FieldSchema {
   autocolumn?: boolean
   throughFrom?: string
   throughTo?: string
+  formula?: string
+  formulaType?: string
   main?: boolean
   meta?: {
     toTable: string
@@ -46,6 +48,7 @@ export interface Table extends Base {
   schema: TableSchema
   primaryDisplay?: string
   sourceId?: string
+  relatedFormula?: string[]
   constrained?: string[]
 }
 

packages/server/src/utilities/rowProcessor/index.js

@@ -180,6 +180,8 @@ function processAutoColumn(
 }
 exports.processAutoColumn = processAutoColumn
 
+exports.processFormulas = processFormulas
+
 /**
  * This will coerce a value to the correct types based on the type transform map
  * @param {object} row The value to coerce
@@ -244,9 +246,6 @@ exports.inputProcessing = (
     clonedRow._rev = row._rev
   }
 
-  // now process the static formulas
-  clonedRow = processFormulas(table, clonedRow, { dynamic: false })
-
   // handle auto columns - this returns an object like {table, row}
   return processAutoColumn(user, copiedTable, clonedRow, opts)
 }

packages/server/src/utilities/rowProcessor/utils.js

@@ -4,10 +4,15 @@ const { processStringSync } = require("@budibase/string-templates")
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */
-exports.processFormulas = (table, rows, { dynamic } = { dynamic: true }) => {
+exports.processFormulas = (
+  table,
+  rows,
+  { dynamic, contextRows } = { dynamic: true }
+) => {
   const single = !Array.isArray(rows)
   if (single) {
     rows = [rows]
+    contextRows = contextRows ? [contextRows] : contextRows
   }
   for (let [column, schema] of Object.entries(table.schema)) {
     const isStatic = schema.formulaType === FormulaTypes.STATIC
@@ -19,10 +24,14 @@ exports.processFormulas = (table, rows, { dynamic } = { dynamic: true }) => {
       continue
     }
     // iterate through rows and process formula
-    rows = rows.map(row => ({
-      ...row,
-      [column]: processStringSync(schema.formula, row),
-    }))
+    for (let i = 0; i < rows.length; i++) {
+      let row = rows[i]
+      let context = contextRows ? contextRows[i] : row
+      rows[i] = {
+        ...row,
+        [column]: processStringSync(schema.formula, context),
+      }
+    }
   }
   return single ? rows[0] : rows
 }

packages/string-templates/src/index.cjs

@@ -15,6 +15,7 @@ module.exports.processStringSync = templates.processStringSync
 module.exports.processObjectSync = templates.processObjectSync
 module.exports.processString = templates.processString
 module.exports.processObject = templates.processObject
+module.exports.doesContainString = templates.doesContainString
 
 /**
  * Use vm2 to run JS scripts in a node env

packages/string-templates/src/index.js

@@ -3,6 +3,7 @@ const { registerAll } = require("./helpers/index")
 const processors = require("./processors")
 const { atob, btoa } = require("./utilities")
 const manifest = require("../manifest.json")
+const { FIND_HBS_REGEX } = require("./utilities")
 
 const hbsInstance = handlebars.create()
 registerAll(hbsInstance)
@@ -26,7 +27,7 @@ function testObject(object) {
  * @param {object|array} object The input structure which is to be recursed, it is important to note that
  * if the structure contains any cycles then this will fail.
  * @param {object} context The context that handlebars should fill data from.
- * @param {object} opts optional - specify some options for processing.
+ * @param {object|null} opts optional - specify some options for processing.
  * @returns {Promise<object|array>} The structure input, as fully updated as possible.
  */
 module.exports.processObject = async (object, context, opts) => {
@@ -57,7 +58,7 @@ module.exports.processObject = async (object, context, opts) => {
  * then nothing will occur.
  * @param {string} string The template string which is the filled from the context object.
  * @param {object} context An object of information which will be used to enrich the string.
- * @param {object} opts optional - specify some options for processing.
+ * @param {object|null} opts optional - specify some options for processing.
  * @returns {Promise<string>} The enriched string, all templates should have been replaced if they can be.
  */
 module.exports.processString = async (string, context, opts) => {
@@ -71,7 +72,7 @@ module.exports.processString = async (string, context, opts) => {
  * @param {object|array} object The input structure which is to be recursed, it is important to note that
  * if the structure contains any cycles then this will fail.
  * @param {object} context The context that handlebars should fill data from.
- * @param {object} opts optional - specify some options for processing.
+ * @param {object|null} opts optional - specify some options for processing.
  * @returns {object|array} The structure input, as fully updated as possible.
  */
 module.exports.processObjectSync = (object, context, opts) => {
@@ -92,7 +93,7 @@ module.exports.processObjectSync = (object, context, opts) => {
  * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call.
  * @param {string} string The template string which is the filled from the context object.
  * @param {object} context An object of information which will be used to enrich the string.
- * @param {object} opts optional - specify some options for processing.
+ * @param {object|null} opts optional - specify some options for processing.
  * @returns {string} The enriched string, all templates should have been replaced if they can be.
  */
 module.exports.processStringSync = (string, context, opts) => {
@@ -221,3 +222,30 @@ module.exports.decodeJSBinding = handlebars => {
   }
   return atob(match[1])
 }
+
+/**
+ * This function looks in the supplied template for handlebars instances, if they contain
+ * JS the JS will be decoded and then the supplied string will be looked for. For example
+ * if the template "Hello, your name is {{ related }}" this function would return that true
+ * for the string "related" but not for "name" as it is not within the handlebars statement.
+ * @param {string} template A template string to search for handlebars instances.
+ * @param {string} string The word or sentence to search for.
+ * @returns {boolean} The this return true if the string is found, false if not.
+ */
+module.exports.doesContainString = (template, string) => {
+  let regexp = new RegExp(FIND_HBS_REGEX)
+  let matches = template.match(regexp)
+  if (matches == null) {
+    return false
+  }
+  for (let match of matches) {
+    let hbs = match
+    if (exports.isJSBinding(match)) {
+      hbs = exports.decodeJSBinding(match)
+    }
+    if (hbs.includes(string)) {
+      return true
+    }
+  }
+  return false
+}

packages/string-templates/src/index.mjs

@@ -15,6 +15,7 @@ export const processStringSync = templates.processStringSync
 export const processObjectSync = templates.processObjectSync
 export const processString = templates.processString
 export const processObject = templates.processObject
+export const doesContainString = templates.doesContainString
 
 /**
  * Use polyfilled vm to run JS scripts in a browser Env

packages/string-templates/test/basic.spec.js

@@ -4,6 +4,8 @@ const {
   isValid,
   makePropSafe,
   getManifest,
+  encodeJSBinding,
+  doesContainString,
 } = require("../src/index.cjs")
 
 describe("Test that the string processing works correctly", () => {
@@ -157,3 +159,20 @@ describe("check full stops that are safe", () => {
     expect(output).toEqual("1")
   })
 })
+
+describe("check does contain string function", () => {
+  it("should work for a simple case", () => {
+    const hbs = "hello {{ name }}"
+    expect(doesContainString(hbs, "name")).toEqual(true)
+  })
+
+  it("should reject a case where its in the string, but not the handlebars", () => {
+    const hbs = "hello {{ name }}"
+    expect(doesContainString(hbs, "hello")).toEqual(false)
+  })
+
+  it("should handle if its in javascript", () => {
+    const js = encodeJSBinding(`return $("foo")`)
+    expect(doesContainString(js, "foo")).toEqual(true)
+  })
+})