Fixing issues with inability to search.

packages/client/yarn.lock

@@ -23,113 +23,6 @@
     chalk "^2.0.0"
     js-tokens "^4.0.0"
 
-<<<<<<< HEAD
-=======
-"@budibase/bbui@^0.9.46":
-  version "0.9.46"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.46.tgz#3109666b618daa65b29d1c7c45549420c62e6489"
-  integrity sha512-PRW8kR9+QrMiom6hVzisMYd268dj03ojC0ruzEkDhKMONg2I021ST62hzKXdb7zh5LgoYXtapmM9qsKwoHfkPg==
-  dependencies:
-    "@adobe/spectrum-css-workflow-icons" "^1.2.1"
-    "@spectrum-css/actionbutton" "^1.0.1"
-    "@spectrum-css/actiongroup" "^1.0.1"
-    "@spectrum-css/avatar" "^3.0.2"
-    "@spectrum-css/button" "^3.0.1"
-    "@spectrum-css/buttongroup" "^3.0.2"
-    "@spectrum-css/checkbox" "^3.0.2"
-    "@spectrum-css/dialog" "^3.0.1"
-    "@spectrum-css/divider" "^1.0.1"
-    "@spectrum-css/dropzone" "^3.0.2"
-    "@spectrum-css/fieldgroup" "^3.0.2"
-    "@spectrum-css/fieldlabel" "^3.0.1"
-    "@spectrum-css/icon" "^3.0.1"
-    "@spectrum-css/illustratedmessage" "^3.0.2"
-    "@spectrum-css/inputgroup" "^3.0.2"
-    "@spectrum-css/label" "^2.0.10"
-    "@spectrum-css/link" "^3.1.1"
-    "@spectrum-css/menu" "^3.0.1"
-    "@spectrum-css/modal" "^3.0.1"
-    "@spectrum-css/pagination" "^3.0.3"
-    "@spectrum-css/picker" "^1.0.1"
-    "@spectrum-css/popover" "^3.0.1"
-    "@spectrum-css/progressbar" "^1.0.2"
-    "@spectrum-css/progresscircle" "^1.0.2"
-    "@spectrum-css/radio" "^3.0.2"
-    "@spectrum-css/search" "^3.0.2"
-    "@spectrum-css/sidenav" "^3.0.2"
-    "@spectrum-css/statuslight" "^3.0.2"
-    "@spectrum-css/switch" "^1.0.2"
-    "@spectrum-css/table" "^3.0.1"
-    "@spectrum-css/tabs" "^3.0.1"
-    "@spectrum-css/tags" "^3.0.2"
-    "@spectrum-css/textfield" "^3.0.1"
-    "@spectrum-css/toast" "^3.0.1"
-    "@spectrum-css/tooltip" "^3.0.3"
-    "@spectrum-css/treeview" "^3.0.2"
-    "@spectrum-css/typography" "^3.0.1"
-    "@spectrum-css/underlay" "^2.0.9"
-    "@spectrum-css/vars" "^3.0.1"
-    dayjs "^1.10.4"
-    svelte-flatpickr "^3.1.0"
-    svelte-portal "^1.0.0"
-
-"@budibase/handlebars-helpers@^0.11.4":
-  version "0.11.4"
-  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.11.4.tgz#8acfa2ee84134f7be4b2906e710fce6d25c5d000"
-  integrity sha512-AJNJYlJnxZmn9QJ8tBl8nrm4YxbwHP4AR0pbiVGK+EoOylkNBlUnZ/QDL1VyqM5fTkAE/Z2IZVLKrrG3kxuWLA==
-  dependencies:
-    arr-flatten "^1.1.0"
-    array-sort "^0.1.4"
-    define-property "^1.0.0"
-    extend-shallow "^3.0.2"
-    "falsey" "^0.3.2"
-    for-in "^1.0.2"
-    for-own "^1.0.0"
-    get-object "^0.2.0"
-    get-value "^2.0.6"
-    handlebars "^4.0.11"
-    handlebars-utils "^1.0.6"
-    has-value "^1.0.0"
-    helper-date "^1.0.1"
-    helper-markdown "^1.0.0"
-    helper-md "^0.2.2"
-    html-tag "^2.0.0"
-    is-even "^1.0.0"
-    is-glob "^4.0.0"
-    is-number "^4.0.0"
-    kind-of "^6.0.0"
-    logging-helpers "^1.0.0"
-    micromatch "^3.1.4"
-    relative "^3.0.2"
-    striptags "^3.1.0"
-    to-gfm-code-block "^0.1.1"
-    year "^0.2.1"
-
-"@budibase/standard-components@^0.9.46":
-  version "0.9.46"
-  resolved "https://registry.yarnpkg.com/@budibase/standard-components/-/standard-components-0.9.46.tgz#a31a253ca51a2029c3aaf5d8aca5c953358e1d67"
-  integrity sha512-QjW4tukMw4Xa477wGTle2UPz85ygodQ3KG+WEdPAWKq7j0IDv0Fad0oDmWtzLvGxxB+AiRbEnM6T1QV6X1ItCA==
-  dependencies:
-    "@budibase/bbui" "^0.9.46"
-    "@spectrum-css/page" "^3.0.1"
-    "@spectrum-css/vars" "^3.0.1"
-    apexcharts "^3.22.1"
-    dayjs "^1.10.5"
-    svelte-apexcharts "^1.0.2"
-    svelte-flatpickr "^3.1.0"
-
-"@budibase/string-templates@^0.9.46":
-  version "0.9.46"
-  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.9.46.tgz#e43f87513977879a892ae52f3941d3320cb9ff88"
-  integrity sha512-yOVS7Y/QLATj31QuBu8KP78Oyzhs60V09JEQKa7n4vRP8TBemcev/LFShln8Iiv70YZSpdJviguQuJ6Ow0aUNA==
-  dependencies:
-    "@budibase/handlebars-helpers" "^0.11.4"
-    dayjs "^1.10.4"
-    handlebars "^4.7.6"
-    handlebars-utils "^1.0.6"
-    lodash "^4.17.20"
-
->>>>>>> 81e794065761192883767a80679d7d94d67afcc2
 "@rollup/plugin-commonjs@^18.0.0":
   version "18.0.0"
   resolved "https://registry.yarnpkg.com/@rollup/plugin-commonjs/-/plugin-commonjs-18.0.0.tgz#50dc7518b5aa9e66a270e529ea85115d269825c4"

packages/server/src/api/controllers/row/external.js

@@ -1,17 +1,7 @@
-const CouchDB = require("../../../db")
 const { makeExternalQuery } = require("./utils")
 const { DataSourceOperation, SortDirection } = require("../../../constants")
-
-async function getTable(appId, datasourceId, tableName) {
-  const db = new CouchDB(appId)
-  const datasource = await db.get(datasourceId)
-  if (!datasource || !datasource.entities) {
-    throw "Datasource is not configured fully."
-  }
-  return Object.values(datasource.entities).find(
-    entity => entity.name === tableName
-  )
-}
+const { getExternalTable } = require("../table/utils")
+const { breakExternalTableId } = require("../../../integrations/utils")
 
 function inputProcessing(row, table) {
   if (!row) {
@@ -69,10 +59,8 @@ async function handleRequest(
   tableId,
   { id, row, filters, sort, paginate } = {}
 ) {
-  const parts = tableId.split("_")
-  let tableName = parts.pop()
-  let datasourceId = parts.join("_")
-  const table = await getTable(appId, datasourceId, tableName)
+  let { datasourceId, tableName } = breakExternalTableId(tableId)
+  const table = await getExternalTable(appId, datasourceId, tableName)
   if (!table) {
     throw `Unable to process query, table "${tableName}" not defined.`
   }
@@ -205,14 +193,16 @@ exports.search = async ctx => {
       [params.sort]: direction,
     }
   }
-  return handleRequest(appId, DataSourceOperation.READ, tableId, {
+  const rows = await handleRequest(appId, DataSourceOperation.READ, tableId, {
     filters: query,
     sort,
     paginate: paginateObj,
   })
+  // need wrapper object for bookmarks etc when paginating
+  return { rows }
 }
 
-exports.validate = async ctx => {
+exports.validate = async () => {
   // can't validate external right now - maybe in future
   return { valid: true }
 }

packages/server/src/api/controllers/row/index.js

@@ -1,9 +1,9 @@
 const internal = require("./internal")
 const external = require("./external")
-const { DocumentTypes } = require("../../../db/utils")
+const { isExternalTable } = require("../../../integrations/utils")
 
 function pickApi(tableId) {
-  if (tableId.includes(DocumentTypes.DATASOURCE)) {
+  if (isExternalTable(tableId)) {
     return external
   }
   return internal

packages/server/src/api/controllers/row/internal.js

@@ -301,7 +301,7 @@ exports.search = async ctx => {
     response.rows = await outputProcessing(appId, table, response.rows)
   }
 
-  ctx.body = response
+  return response
 }
 
 exports.validate = async ctx => {

packages/server/src/api/controllers/table/index.js

@@ -6,11 +6,17 @@ const {
   getTableParams,
   generateTableID,
   getDatasourceParams,
-  DocumentTypes,
   BudibaseInternalDB,
 } = require("../../../db/utils")
 const { FieldTypes } = require("../../../constants")
-const { TableSaveFunctions } = require("./utils")
+const {
+  TableSaveFunctions,
+  getExternalTable
+} = require("./utils")
+const {
+  isExternalTable,
+  breakExternalTableId
+} = require("../../../integrations/utils")
 
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
@@ -44,7 +50,13 @@ exports.fetch = async function (ctx) {
 
 exports.find = async function (ctx) {
   const db = new CouchDB(ctx.appId)
-  ctx.body = await db.get(ctx.params.id)
+  const tableId = ctx.params.id
+  if (isExternalTable(tableId)) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    ctx.body = await getExternalTable(ctx.appId, datasourceId, tableName)
+  } else {
+    ctx.body = await db.get(ctx.params.id)
+  }
 }
 
 exports.save = async function (ctx) {

packages/server/src/api/controllers/table/utils.js

@@ -204,4 +204,15 @@ class TableSaveFunctions {
   }
 }
 
+exports.getExternalTable = async (appId, datasourceId, tableName) => {
+  const db = new CouchDB(appId)
+  const datasource = await db.get(datasourceId)
+  if (!datasource || !datasource.entities) {
+    throw "Datasource is not configured fully."
+  }
+  return Object.values(datasource.entities).find(
+    entity => entity.name === tableName
+  )
+}
+
 exports.TableSaveFunctions = TableSaveFunctions

packages/server/src/integrations/postgres.js

@@ -3,6 +3,7 @@ const { FIELD_TYPES } = require("./Integration")
 const Sql = require("./base/sql")
 const { FieldTypes } = require("../constants")
 const { SEPARATOR } = require("@budibase/auth/db")
+const { buildExternalTableId } = require("./utils")
 
 const SCHEMA = {
   docs: "https://node-postgres.com",
@@ -137,7 +138,7 @@ class PostgresIntegration extends Sql {
       // table key doesn't exist yet
       if (!tables[tableName]) {
         tables[tableName] = {
-          _id: `${datasourceId}${SEPARATOR}${tableName}`,
+          _id: buildExternalTableId(datasourceId, tableName),
           // TODO: this needs to accommodate composite keys
           primary: keys,
           name: tableName,

packages/server/src/integrations/utils.js

@@ -0,0 +1,18 @@
+const { DocumentTypes, SEPARATOR } = require("../db/utils")
+
+const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
+
+exports.isExternalTable = tableId => {
+  return tableId.includes(DocumentTypes.DATASOURCE)
+}
+
+exports.buildExternalTableId = (datasourceId, tableName) => {
+  return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
+}
+
+exports.breakExternalTableId = tableId => {
+  const parts = tableId.split(DOUBLE_SEPARATOR)
+  let tableName = parts.pop()
+  let datasourceId = parts.join(DOUBLE_SEPARATOR)
+  return { datasourceId, tableName }
+}