Moving views into a different location so they don't trigger tree creation and attempting to use in memory pouchDB to run views on the fly.

packages/server/src/api/controllers/row/internal.js

@@ -5,6 +5,7 @@ const {
   generateRowID,
   DocumentTypes,
   InternalTables,
+  generateMemoryViewID,
 } = require("../../../db/utils")
 const userController = require("../user")
 const {
@@ -16,6 +17,8 @@ const { isEqual } = require("lodash")
 const { validate, findRow } = require("./utils")
 const { fullSearch, paginatedSearch } = require("./internalSearch")
 const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
+const inMemoryViews = require("../../../db/inMemoryView")
+const env = require("../../../environment")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -36,6 +39,40 @@ async function storeResponse(ctx, db, row, oldTable, table) {
   return { row, table }
 }
 
+// doesn't do the outputProcessing
+async function getRawTableData(ctx, db, tableId) {
+  let rows
+  if (tableId === InternalTables.USER_METADATA) {
+    await userController.fetchMetadata(ctx)
+    rows = ctx.body
+  } else {
+    const response = await db.allDocs(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+    rows = response.rows.map(row => row.doc)
+  }
+  return rows
+}
+
+async function getView(db, viewName) {
+  let viewInfo
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
+    viewInfo = designDoc.views[viewName]
+  } else {
+    viewInfo = await db.get(generateMemoryViewID(viewName))
+    if (viewInfo) {
+      viewInfo = viewInfo.view
+    }
+  }
+  if (!viewInfo) {
+    throw "View does not exist."
+  }
+  return viewInfo
+}
+
 exports.patch = async ctx => {
   const appId = ctx.appId
   const db = new CouchDB(appId)
@@ -139,15 +176,28 @@ exports.fetchView = async ctx => {
 
   const db = new CouchDB(appId)
   const { calculation, group, field } = ctx.query
-  const designDoc = await db.get("_design/database")
-  const viewInfo = designDoc.views[viewName]
+  const viewInfo = await getView(db, viewName)
   if (!viewInfo) {
     throw "View does not exist."
   }
-  const response = await db.query(`database/${viewName}`, {
-    include_docs: !calculation,
-    group: !!group,
-  })
+  let response
+  // TODO: make sure not self hosted in Cloud
+  if (!env.SELF_HOSTED) {
+    response = await db.query(`database/${viewName}`, {
+      include_docs: !calculation,
+      group: !!group,
+    })
+  } else {
+    const tableId = viewInfo.meta.tableId
+    const data = await getRawTableData(ctx, db, tableId)
+    response = await inMemoryViews.runView(
+      appId,
+      viewInfo,
+      calculation,
+      group,
+      data
+    )
+  }
 
   let rows
   if (!calculation) {
@@ -191,19 +241,9 @@ exports.fetch = async ctx => {
   const appId = ctx.appId
   const db = new CouchDB(appId)
 
-  let rows,
-    table = await db.get(ctx.params.tableId)
-  if (ctx.params.tableId === InternalTables.USER_METADATA) {
-    await userController.fetchMetadata(ctx)
-    rows = ctx.body
-  } else {
-    const response = await db.allDocs(
-      getRowParams(ctx.params.tableId, null, {
-        include_docs: true,
-      })
-    )
-    rows = response.rows.map(row => row.doc)
-  }
+  const tableId = ctx.params.tableId
+  let table = await db.get(tableId)
+  let rows = await getRawTableData(ctx, db, tableId)
   return outputProcessing(ctx, table, rows)
 }
 

packages/server/src/api/controllers/view/index.js

@@ -3,14 +3,27 @@ const viewTemplate = require("./viewBuilder")
 const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
 const { fetchView } = require("../row")
-const { ViewNames } = require("../../../db/utils")
-
-const controller = {
-  fetch: async ctx => {
-    const db = new CouchDB(ctx.appId)
+const {
+  ViewNames,
+  generateMemoryViewID,
+  getMemoryViewParams,
+} = require("../../../db/utils")
+const env = require("../../../environment")
+
+async function getView(db, viewName) {
+  if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
-    const response = []
+    return designDoc.views[viewName]
+  } else {
+    const viewDoc = await db.get(generateMemoryViewID(viewName))
+    return viewDoc.view
+  }
+}
 
+async function getViews(db) {
+  const response = []
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
     for (let name of Object.keys(designDoc.views)) {
       // Only return custom views, not built ins
       if (Object.values(ViewNames).indexOf(name) !== -1) {
@@ -21,30 +34,91 @@ const controller = {
         ...designDoc.views[name],
       })
     }
-
-    ctx.body = response
-  },
-  save: async ctx => {
-    const db = new CouchDB(ctx.appId)
-    const { originalName, ...viewToSave } = ctx.request.body
-    const designDoc = await db.get("_design/database")
-    const view = viewTemplate(viewToSave)
-
-    if (!viewToSave.name) {
-      ctx.throw(400, "Cannot create view without a name")
+  } else {
+    const views = (
+      await db.allDocs(
+        getMemoryViewParams({
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+    for (let viewDoc of views) {
+      response.push({
+        name: viewDoc.name,
+        ...viewDoc.view,
+      })
     }
+  }
+  return response
+}
 
+async function saveView(db, originalName, viewToSave, viewTemplate) {
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
     designDoc.views = {
       ...designDoc.views,
-      [viewToSave.name]: view,
+      [viewToSave.name]: viewTemplate,
     }
-
     // view has been renamed
     if (originalName) {
       delete designDoc.views[originalName]
     }
+    await db.put(designDoc)
+  } else {
+    const id = generateMemoryViewID(viewToSave.name)
+    const originalId = originalName ? generateMemoryViewID(originalName) : null
+    const viewDoc = {
+      _id: id,
+      view: viewTemplate,
+      name: viewToSave.name,
+      tableId: viewTemplate.meta.tableId,
+    }
+    try {
+      const old = await db.get(id)
+      if (originalId) {
+        const originalDoc = await db.get(originalId)
+        await db.remove(originalDoc._id, originalDoc._rev)
+      }
+      if (old && old._rev) {
+        viewDoc._rev = old._rev
+      }
+    } catch (err) {
+      // didn't exist, just skip
+    }
+    await db.put(viewDoc)
+  }
+}
 
+async function deleteView(db, viewName) {
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
+    const view = designDoc.views[viewName]
+    delete designDoc.views[viewName]
     await db.put(designDoc)
+    return view
+  } else {
+    const id = generateMemoryViewID(viewName)
+    const viewDoc = await db.get(id)
+    await db.remove(viewDoc._id, viewDoc._rev)
+    return viewDoc.view
+  }
+}
+
+const controller = {
+  fetch: async ctx => {
+    const db = new CouchDB(ctx.appId)
+    ctx.body = await getViews(db)
+  },
+  save: async ctx => {
+    const db = new CouchDB(ctx.appId)
+    const { originalName, ...viewToSave } = ctx.request.body
+    const view = viewTemplate(viewToSave)
+
+    if (!viewToSave.name) {
+      ctx.throw(400, "Cannot create view without a name")
+    }
+
+    await saveView(db, originalName, viewToSave, view)
 
     // add views to table document
     const table = await db.get(ctx.request.body.tableId)
@@ -53,11 +127,9 @@ const controller = {
       view.meta.schema = table.schema
     }
     table.views[viewToSave.name] = view.meta
-
     if (originalName) {
       delete table.views[originalName]
     }
-
     await db.put(table)
 
     ctx.body = {
@@ -67,13 +139,8 @@ const controller = {
   },
   destroy: async ctx => {
     const db = new CouchDB(ctx.appId)
-    const designDoc = await db.get("_design/database")
     const viewName = decodeURI(ctx.params.viewName)
-    const view = designDoc.views[viewName]
-    delete designDoc.views[viewName]
-
-    await db.put(designDoc)
-
+    const view = await deleteView(db, viewName)
     const table = await db.get(view.meta.tableId)
     delete table.views[viewName]
     await db.put(table)
@@ -82,10 +149,9 @@ const controller = {
   },
   exportView: async ctx => {
     const db = new CouchDB(ctx.appId)
-    const designDoc = await db.get("_design/database")
     const viewName = decodeURI(ctx.query.view)
+    const view = await getView(db, viewName)
 
-    const view = designDoc.views[viewName]
     const format = ctx.query.format
     if (!format) {
       ctx.throw(400, "Format must be specified, either csv or json")

packages/server/src/db/inMemoryView.js

@@ -0,0 +1,38 @@
+const PouchDB = require("pouchdb")
+const memory = require("pouchdb-adapter-memory")
+
+PouchDB.plugin(memory)
+const Pouch = PouchDB.defaults({
+  prefix: undefined,
+  adapter: "memory",
+})
+
+exports.runView = async (appId, view, calculation, group, data) => {
+  // appId doesn't really do anything since its all in memory
+  // use it just incase multiple databases at the same time
+  const db = new Pouch(appId)
+  await db.put({
+    _id: "_design/database",
+    views: {
+      runner: view,
+    },
+  })
+  // write all the docs to the in memory Pouch
+  await db.bulkDocs(data)
+  const response = await db.query(`database/runner`, {
+    include_docs: !calculation,
+    group: !!group,
+  })
+  // need to fix the revs to be totally accurate
+  for (let row of response.rows) {
+    if (!row._rev || !row._id) {
+      continue
+    }
+    const found = data.find(possible => possible._id === row._id)
+    if (found) {
+      row._rev = found._rev
+    }
+  }
+  await db.destroy()
+  return response
+}

packages/server/src/db/utils.js

@@ -39,6 +39,7 @@ const DocumentTypes = {
   QUERY: "query",
   DEPLOYMENTS: "deployments",
   METADATA: "metadata",
+  MEM_VIEW: "view",
 }
 
 const ViewNames = {
@@ -348,6 +349,14 @@ exports.getMetadataParams = (type, entityId = null, otherProps = {}) => {
   return getDocParams(DocumentTypes.METADATA, docId, otherProps)
 }
 
+exports.generateMemoryViewID = viewName => {
+  return `${DocumentTypes.MEM_VIEW}${SEPARATOR}${viewName}`
+}
+
+exports.getMemoryViewParams = (otherProps = {}) => {
+  return getDocParams(DocumentTypes.MEM_VIEW, null, otherProps)
+}
+
 /**
  * This can be used with the db.allDocs to get a list of IDs
  */

packages/server/src/environment.js

@@ -66,3 +66,10 @@ module.exports = {
     return !isDev()
   },
 }
+
+// convert any strings to numbers if required, like "0" would be true otherwise
+for (let [key, value] of Object.entries(module.exports)) {
+  if (typeof value === "string" && !isNaN(parseInt(value))) {
+    module.exports[key] = parseInt(value)
+  }
+}