Rewriting search to use the new couchdb 3.0 search functionality.

packages/client/src/api/tables.js

@@ -21,14 +21,16 @@ export const fetchTableData = async tableId => {
  * Perform a mango query against an internal table
  * @param {String} tableId - id of the table to search
  * @param {Object} search - Mango Compliant search object
+ * @param {Object} pagination - the pagination controls
  */
 export const searchTableData = async ({ tableId, search, pagination }) => {
-  const rows = await API.post({
+  const output = await API.post({
     url: `/api/${tableId}/rows/search`,
     body: {
       query: search,
       pagination,
     },
   })
-  return await enrichRows(rows, tableId)
+  output.rows = await enrichRows(output.rows, tableId)
+  return output
 }

packages/server/src/api/controllers/application.js

@@ -5,7 +5,7 @@ const packageJson = require("../../../package.json")
 const {
   createLinkView,
   createRoutingView,
-  createFulltextSearchIndex,
+  createAllSearchIndex,
 } = require("../../db/views/staticViews")
 const {
   getTemplateStream,
@@ -95,7 +95,7 @@ async function createInstance(template) {
   // add view for linked rows
   await createLinkView(appId)
   await createRoutingView(appId)
-  await createFulltextSearchIndex(appId)
+  await createAllSearchIndex(appId)
 
   // replicate the template data to the instance DB
   // this is currently very hard to test, downloading and importing template files

packages/server/src/api/controllers/row.js

@@ -17,6 +17,7 @@ const {
 const { FieldTypes } = require("../../constants")
 const { isEqual } = require("lodash")
 const { cloneDeep } = require("lodash/fp")
+const searchController = require("./search")
 
 const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}`
 
@@ -259,39 +260,40 @@ exports.search = async function(ctx) {
   const db = new CouchDB(appId)
   const {
     query,
-    pagination: { pageSize = 10, page },
+    pagination: { pageSize = 10, bookmark },
   } = ctx.request.body
+  const tableId = ctx.params.tableId
+
+  const queryBuilder = new searchController.QueryBuilder(appId)
+    .setLimit(pageSize)
+    .addTable(tableId)
+  if (bookmark) {
+    queryBuilder.setBookmark(bookmark)
+  }
 
   // make all strings a starts with operation rather than pure equality
   for (const [key, queryVal] of Object.entries(query)) {
     if (typeof queryVal === "string") {
-      query[key] = {
+      queryBuilder.addString(key, queryVal)
-        $gt: queryVal,
+    } else {
-        $lt: `${queryVal}\uffff`,
+      queryBuilder.addEqual(key, queryVal)
-      }
     }
   }
 
-  // pure equality for table
+  const response = await searchController.search(queryBuilder.complete())
-  query.tableId = ctx.params.tableId
-  const response = await db.find({
-    selector: query,
-    limit: pageSize,
-    skip: pageSize * page,
-  })
-
-  const rows = response.docs
 
   // delete passwords from users
-  if (query.tableId === ViewNames.USERS) {
+  if (tableId === ViewNames.USERS) {
-    for (let row of rows) {
+    for (let row of response.rows) {
       delete row.password
     }
   }
 
-  const table = await db.get(ctx.params.tableId)
+  const table = await db.get(tableId)
-
+  ctx.body = {
-  ctx.body = await outputProcessing(appId, table, rows)
+    rows: await outputProcessing(appId, table, response.rows),
+    bookmark: response.bookmark,
+  }
 }
 
 exports.fetchTableRows = async function(ctx) {

packages/server/src/api/controllers/search.js

@@ -0,0 +1,138 @@
+const fetch = require("node-fetch")
+const { SearchIndexes } = require("../../db/utils")
+const { checkSlashesInUrl } = require("../../utilities")
+const env = require("../../environment")
+
+function buildSearchUrl(
+  appId,
+  query,
+  bookmark = null,
+  limit = 50,
+  includeDocs = true
+) {
+  let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search`
+  url += `/${SearchIndexes.ROWS}?q=${query}`
+  if (includeDocs) {
+    url += "&include_docs=true"
+  }
+  if (limit) {
+    url += `&limit=${limit}`
+  }
+  if (bookmark) {
+    url += `&bookmark=${bookmark}`
+  }
+  return checkSlashesInUrl(url)
+}
+
+class QueryBuilder {
+  constructor(appId, base) {
+    this.appId = appId
+    this.query = {
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {},
+      meta: {},
+      ...base,
+    }
+    this.limit = 50
+    this.bookmark = null
+  }
+
+  setLimit(limit) {
+    this.limit = limit
+    return this
+  }
+
+  setBookmark(bookmark) {
+    this.bookmark = bookmark
+    return this
+  }
+
+  addString(key, partial) {
+    this.query.string[key] = partial
+    return this
+  }
+
+  addFuzzy(key, fuzzy) {
+    this.query.fuzzy[key] = fuzzy
+    return this
+  }
+
+  addRange(key, low, high) {
+    this.query.range = {
+      low,
+      high,
+    }
+    return this
+  }
+
+  addEqual(key, value) {
+    this.query.equal[key] = value
+    return this
+  }
+
+  addTable(tableId) {
+    this.query.equal.tableId = tableId
+    return this
+  }
+
+  complete() {
+    let output = ""
+    function build(structure, queryFn) {
+      for (let [key, value] of Object.entries(structure)) {
+        if (output.length !== 0) {
+          output += " AND "
+        }
+        output += queryFn(key, value)
+      }
+    }
+
+    if (this.query.string) {
+      build(this.query.string, (key, value) => `${key}:${value}*`)
+    }
+    if (this.query.number) {
+      build(this.query.number, (key, value) =>
+        value.length == null
+          ? `${key}:${value}`
+          : `${key}:[${value[0]} TO ${value[1]}]`
+      )
+    }
+    if (this.query.fuzzy) {
+      build(this.query.fuzzy, (key, value) => `${key}:${value}~`)
+    }
+    return buildSearchUrl(this.appId, output, this.bookmark, this.limit)
+  }
+}
+
+exports.QueryBuilder = QueryBuilder
+
+exports.search = async query => {
+  const response = await fetch(query, {
+    method: "GET",
+  })
+  const json = await response.json()
+  let output = {
+    rows: [],
+  }
+  if (json.rows != null && json.rows.length > 0) {
+    output.rows = json.rows.map(row => row.doc)
+  }
+  if (json.bookmark) {
+    output.bookmark = json.bookmark
+  }
+  return output
+}
+
+exports.rowSearch = async ctx => {
+  // this can't be done through pouch, have to reach for trusty node-fetch
+  const appId = ctx.user.appId
+  const bookmark = ctx.params.bookmark
+  let url
+  if (ctx.params.query) {
+    url = new QueryBuilder(appId, ctx.params.query, bookmark).complete()
+  } else if (ctx.params.raw) {
+    url = buildSearchUrl(appId, ctx.params.raw, bookmark)
+  }
+  ctx.body = await exports.search(url)
+}

packages/server/src/api/controllers/static/index.js

@@ -2,6 +2,7 @@ require("svelte/register")
 
 const send = require("koa-send")
 const { resolve, join } = require("../../../utilities/centralPath")
+const { checkSlashesInUrl } = require("../../../utilities")
 const fetch = require("node-fetch")
 const uuid = require("uuid")
 const { prepareUpload } = require("../deploy/utils")
@@ -28,10 +29,7 @@ function objectStoreUrl() {
 
 function internalObjectStoreUrl() {
   if (env.SELF_HOSTED) {
-    return (env.MINIO_URL + OBJ_STORE_DIRECTORY).replace(
+    return checkSlashesInUrl(env.MINIO_URL + OBJ_STORE_DIRECTORY)
-      /(https?:\/\/)|(\/)+/g,
-      "$1$2"
-    )
   } else {
     return BB_CDN
   }

packages/server/src/api/routes/search.js

@@ -0,0 +1,8 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/search")
+
+const router = Router()
+
+router.get("/api/search/rows", controller.rowSearch)
+
+module.exports = router

packages/server/src/db/utils.js

@@ -37,11 +37,16 @@ const ViewNames = {
   USERS: "ta_users",
 }
 
+const SearchIndexes = {
+  ROWS: "rows",
+}
+
 exports.StaticDatabases = StaticDatabases
 exports.ViewNames = ViewNames
 exports.DocumentTypes = DocumentTypes
 exports.SEPARATOR = SEPARATOR
 exports.UNICODE_MAX = UNICODE_MAX
+exports.SearchIndexes = SearchIndexes
 
 exports.getQueryIndex = viewName => {
   return `database/${viewName}`

packages/server/src/db/views/staticViews.js

@@ -1,5 +1,10 @@
 const CouchDB = require("../index")
-const { DocumentTypes, SEPARATOR, ViewNames } = require("../utils")
+const {
+  DocumentTypes,
+  SEPARATOR,
+  ViewNames,
+  SearchIndexes,
+} = require("../utils")
 const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
 
 /**************************************************
@@ -73,30 +78,41 @@ exports.createRoutingView = async appId => {
   await db.put(designDoc)
 }
 
-exports.createFulltextSearchIndex = async appId => {
+async function searchIndex(appId, indexName, fnString) {
   const db = new CouchDB(appId)
   const designDoc = await db.get("_design/database")
   designDoc.indexes = {
-    rows: {
+    [indexName]: {
-      index: function(doc) {
+      index: fnString,
-        // eslint-disable-next-line no-undef
-        index("id", doc._id)
-        function idx(obj, prev = "") {
-          for (let key of Object.keys(obj)) {
-            let prevKey = prev !== "" ? `${prev}.${key}` : key
-            if (typeof obj[key] !== "object") {
-              // eslint-disable-next-line no-undef
-              index(prevKey, obj[key], { store: true })
-            } else {
-              idx(obj[key], prevKey)
-            }
-          }
-        }
-        if (doc._id.startsWith("ro_")) {
-          idx(doc)
-        }
-      }.toString(),
     },
   }
   await db.put(designDoc)
 }
+
+exports.createAllSearchIndex = async appId => {
+  await searchIndex(
+    appId,
+    SearchIndexes.ROWS,
+    function(doc) {
+      function idx(input, prev) {
+        for (let key of Object.keys(input)) {
+          const idxKey = prev != null ? `${prev}.${key}` : key
+          if (key === "_id" || key === "_rev") {
+            continue
+          }
+          if (typeof input[key] !== "object") {
+            // eslint-disable-next-line no-undef
+            index(idxKey, input[key], { store: true })
+          } else {
+            idx(input[key], idxKey)
+          }
+        }
+      }
+      if (doc._id.startsWith("ro_")) {
+        // eslint-disable-next-line no-undef
+        index("default", doc._id)
+        idx(doc)
+      }
+    }.toString()
+  )
+}

packages/server/src/utilities/index.js

@@ -106,3 +106,7 @@ exports.getAllApps = async () => {
       .map(({ value }) => value)
   }
 }
+
+exports.checkSlashesInUrl = url => {
+  return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
+}

packages/standard-components/src/Search.svelte

@@ -25,10 +25,11 @@
   let tableDefinition
   let schema
 
-  // pagination
+  let nextBookmark = null
-  let page = 0
+  let bookmark = null
+  let lastBookmark = null
 
-  $: fetchData(table, page)
+  $: fetchData(table, bookmark)
   // omit empty strings
   $: parsedSearch = Object.keys(search).reduce(
     (acc, next) =>
@@ -38,33 +39,38 @@
   $: actions = [
     {
       type: ActionTypes.RefreshDatasource,
-      callback: () => fetchData(table, page),
+      callback: () => fetchData(table, bookmark),
       metadata: { datasource: { type: "table", tableId: table } },
     },
   ]
 
-  async function fetchData(table, page) {
+  async function fetchData(table, mark) {
     if (table) {
       const tableDef = await API.fetchTableDefinition(table)
       schema = tableDef.schema
-      rows = await API.searchTableData({
+      lastBookmark = mark
+      const output = await API.searchTableData({
         tableId: table,
         search: parsedSearch,
         pagination: {
           pageSize,
-          page,
+          bookmark: mark,
         },
       })
+      rows = output.rows
+      nextBookmark = output.bookmark
     }
     loaded = true
   }
 
   function nextPage() {
-    page += 1
+    lastBookmark = bookmark
+    bookmark = nextBookmark
   }
 
   function previousPage() {
-    page -= 1
+    nextBookmark = bookmark
+    bookmark = lastBookmark
   }
 </script>
 
@@ -99,15 +105,15 @@
           secondary
           on:click={() => {
             search = {}
-            page = 0
+            bookmark = null
           }}>
           Reset
         </Button>
         <Button
           primary
           on:click={() => {
-            page = 0
+            bookmark = null
-            fetchData(table, page)
+            fetchData(table, bookmark)
           }}>
           Search
         </Button>
@@ -129,7 +135,7 @@
       {/if}
     {/if}
     <div class="pagination">
-      {#if page > 0}
+      {#if bookmark != null}
         <Button primary on:click={previousPage}>Back</Button>
       {/if}
       {#if rows.length === pageSize}