Merge pull request #2909 from Budibase/fix/mike-fixes

Various minor fixes

packages/auth/src/redis/index.js

@@ -191,6 +191,12 @@ class RedisWrapper {
     }
   }
 
+  async getTTL(key) {
+    const db = this._db
+    const prefixedKey = addDbPrefix(db, key)
+    return CLIENT.ttl(prefixedKey)
+  }
+
   async setExpiry(key, expirySeconds) {
     const db = this._db
     const prefixedKey = addDbPrefix(db, key)

packages/builder/src/builderStore/store/frontend.js

@@ -67,6 +67,14 @@ export const getFrontendStore = () => {
     initialise: async pkg => {
       const { layouts, screens, application, clientLibPath } = pkg
       const components = await fetchComponentLibDefinitions(application.appId)
+      // make sure app isn't locked
+      if (
+        components &&
+        components.status === 400 &&
+        components.message?.includes("lock")
+      ) {
+        throw { ok: false, reason: "locked" }
+      }
       store.update(state => ({
         ...state,
         libraries: application.componentLibraries,

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -11,16 +11,18 @@
   import ICONS from "./icons"
 
   let openDataSources = []
-  $: enrichedDataSources = $datasources.list.map(datasource => {
+  $: enrichedDataSources = Array.isArray($datasources.list)
-    const selected = $datasources.selected === datasource._id
+    ? $datasources.list.map(datasource => {
-    const open = openDataSources.includes(datasource._id)
+        const selected = $datasources.selected === datasource._id
-    const containsSelected = containsActiveEntity(datasource)
+        const open = openDataSources.includes(datasource._id)
-    return {
+        const containsSelected = containsActiveEntity(datasource)
-      ...datasource,
+        return {
-      selected,
+          ...datasource,
-      open: selected || open || containsSelected,
+          selected,
-    }
+          open: selected || open || containsSelected,
-  })
+        }
+      })
+    : []
   $: openDataSource = enrichedDataSources.find(x => x.open)
   $: {
     // Ensure the open data source is always included in the list of open

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -3,7 +3,7 @@
   import { ModalContent, notifications, Body, Layout } from "@budibase/bbui"
   import analytics, { Events } from "analytics"
   import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
-  import { datasources } from "stores/backend"
+  import { datasources, tables } from "stores/backend"
   import { IntegrationNames } from "constants"
   import cloneDeep from "lodash/cloneDeepWith"
 
@@ -21,7 +21,9 @@
 
     let baseName = IntegrationNames[config.type]
     let name =
-      existingTypeCount == 0 ? baseName : `${baseName}-${existingTypeCount + 1}`
+      existingTypeCount === 0
+        ? baseName
+        : `${baseName}-${existingTypeCount + 1}`
 
     datasource.type = "datasource"
     datasource.source = config.type
@@ -37,6 +39,8 @@
       // Create datasource
       const resp = await datasources.save(datasource, datasource.plus)
 
+      // update the tables incase data source plus
+      await tables.fetch()
       await datasources.select(resp._id)
       $goto(`./datasource/${resp._id}`)
       notifications.success(`Datasource updated successfully.`)

packages/builder/src/constants/lucene.js

@@ -44,6 +44,15 @@ export const OperatorOptions = {
   },
 }
 
+export const NoEmptyFilterStrings = [
+  OperatorOptions.StartsWith.value,
+  OperatorOptions.Like.value,
+  OperatorOptions.Equals.value,
+  OperatorOptions.NotEquals.value,
+  OperatorOptions.Contains.value,
+  OperatorOptions.NotContains.value,
+]
+
 /**
  * Returns the valid operator options for a certain data type
  * @param type the data type

packages/builder/src/helpers/lucene.js

@@ -1,3 +1,26 @@
+import { NoEmptyFilterStrings } from "../constants/lucene"
+
+/**
+ * Removes any fields that contain empty strings that would cause inconsistent
+ * behaviour with how backend tables are filtered (no value means no filter).
+ */
+function cleanupQuery(query) {
+  if (!query) {
+    return query
+  }
+  for (let filterField of NoEmptyFilterStrings) {
+    if (!query[filterField]) {
+      continue
+    }
+    for (let [key, value] of Object.entries(query[filterField])) {
+      if (!value || value === "") {
+        delete query[filterField][key]
+      }
+    }
+  }
+  return query
+}
+
 /**
  * Builds a lucene JSON query from the filter structure generated in the builder
  * @param filter the builder filter structure
@@ -76,6 +99,8 @@ export const luceneQuery = (docs, query) => {
   if (!query) {
     return docs
   }
+  // make query consistent first
+  query = cleanupQuery(query)
 
   // Iterates over a set of filters and evaluates a fail function against a doc
   const match = (type, failFn) => doc => {

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -8,7 +8,7 @@
   import NPSFeedbackForm from "components/feedback/NPSFeedbackForm.svelte"
   import { get } from "builderStore/api"
   import { auth, admin } from "stores/portal"
-  import { isActive, goto, layout } from "@roxi/routify"
+  import { isActive, goto, layout, redirect } from "@roxi/routify"
   import Logo from "assets/bb-emblem.svg"
   import { capitalise } from "helpers"
   import UpgradeModal from "../../../../components/upgrade/UpgradeModal.svelte"
@@ -34,7 +34,16 @@
     const pkg = await res.json()
 
     if (res.ok) {
-      await store.actions.initialise(pkg)
+      try {
+        await store.actions.initialise(pkg)
+        // edge case, lock wasn't known to client when it re-directed, or user went directly
+      } catch (err) {
+        if (!err.ok && err.reason === "locked") {
+          $redirect("../../")
+        } else {
+          throw err
+        }
+      }
       await automationStore.actions.fetch()
       await roles.fetch()
       return pkg

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -178,7 +178,12 @@ module External {
         manyRelationships: ManyRelationship[] = []
       for (let [key, field] of Object.entries(table.schema)) {
         // if set already, or not set just skip it
-        if (!row[key] || newRow[key] || field.autocolumn) {
+        if ((!row[key] && row[key] !== "") || newRow[key] || field.autocolumn) {
+          continue
+        }
+        // if its an empty string then it means return the column to null (if possible)
+        if (row[key] === "") {
+          newRow[key] = null
           continue
         }
         // parse floats/numbers

packages/server/src/api/controllers/row/external.js

@@ -2,6 +2,7 @@ const {
   DataSourceOperation,
   SortDirection,
   FieldTypes,
+  NoEmptyFilterStrings,
 } = require("../../../constants")
 const {
   breakExternalTableId,
@@ -11,6 +12,19 @@ const ExternalRequest = require("./ExternalRequest")
 const CouchDB = require("../../../db")
 
 async function handleRequest(appId, operation, tableId, opts = {}) {
+  // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
+  if (opts && opts.filters) {
+    for (let filterField of NoEmptyFilterStrings) {
+      if (!opts.filters[filterField]) {
+        continue
+      }
+      for (let [key, value] of Object.entries(opts.filters[filterField])) {
+        if (!value || value === "") {
+          delete opts.filters[filterField][key]
+        }
+      }
+    }
+  }
   return new ExternalRequest(appId, operation, tableId, opts.datasource).run(
     opts
   )

packages/server/src/constants/index.js

@@ -6,6 +6,29 @@ exports.JobQueues = {
   AUTOMATIONS: "automationQueue",
 }
 
+const FilterTypes = {
+  STRING: "string",
+  FUZZY: "fuzzy",
+  RANGE: "range",
+  EQUAL: "equal",
+  NOT_EQUAL: "notEqual",
+  EMPTY: "empty",
+  NOT_EMPTY: "notEmpty",
+  CONTAINS: "contains",
+  NOT_CONTAINS: "notContains",
+  ONE_OF: "oneOf",
+}
+
+exports.FilterTypes = FilterTypes
+exports.NoEmptyFilterStrings = [
+  FilterTypes.STRING,
+  FilterTypes.FUZZY,
+  FilterTypes.EQUAL,
+  FilterTypes.NOT_EQUAL,
+  FilterTypes.CONTAINS,
+  FilterTypes.NOT_CONTAINS,
+]
+
 exports.FieldTypes = {
   STRING: "string",
   LONGFORM: "longform",

packages/server/src/integrations/base/sql.ts

@@ -55,6 +55,12 @@ function addFilters(
       query = query[fnc](key, "ilike", `${value}%`)
     })
   }
+  if (filters.fuzzy) {
+    iterate(filters.fuzzy, (key, value) => {
+      const fnc = allOr ? "orWhere" : "where"
+      query = query[fnc](key, "ilike", `%${value}%`)
+    })
+  }
   if (filters.range) {
     iterate(filters.range, (key, value) => {
       if (!value.high || !value.low) {
@@ -135,6 +141,12 @@ function buildCreate(
   const { endpoint, body } = json
   let query: KnexQuery = knex(endpoint.entityId)
   const parsedBody = parseBody(body)
+  // make sure no null values in body for creation
+  for (let [key, value] of Object.entries(parsedBody)) {
+    if (value == null) {
+      delete parsedBody[key]
+    }
+  }
   // mysql can't use returning
   if (opts.disableReturning) {
     return query.insert(parsedBody)

packages/server/src/middleware/builder.js

@@ -33,7 +33,7 @@ async function checkDevAppLocks(ctx) {
     return
   }
   if (!(await doesUserHaveLock(appId, ctx.user))) {
-    ctx.throw(403, "User does not hold app lock.")
+    ctx.throw(400, "User does not hold app lock.")
   }
 
   // they do have lock, update it