Adding the ability to migrate from existing in db views to in memory views.

packages/server/src/api/controllers/row/internal.js

@@ -20,6 +20,7 @@ const { fullSearch, paginatedSearch } = require("./internalSearch")
 const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
 const inMemoryViews = require("../../../db/inMemoryView")
 const env = require("../../../environment")
+const { migrateToInMemoryView } = require("../view/utils")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -74,15 +75,33 @@ async function getRawTableData(ctx, db, tableId) {
 
 async function getView(db, viewName) {
   let viewInfo
-  if (env.SELF_HOSTED) {
+  async function getFromDesignDoc() {
     const designDoc = await db.get("_design/database")
     viewInfo = designDoc.views[viewName]
+    return viewInfo
+  }
+  let migrate = false
+  if (env.SELF_HOSTED) {
+    viewInfo = await getFromDesignDoc()
   } else {
-    viewInfo = await db.get(generateMemoryViewID(viewName))
+    try {
-    if (viewInfo) {
+      viewInfo = await db.get(generateMemoryViewID(viewName))
-      viewInfo = viewInfo.view
+      if (viewInfo) {
+        viewInfo = viewInfo.view
+      }
+    } catch (err) {
+      // check if it can be retrieved from design doc (needs migrated)
+      if (err.status !== 404) {
+        viewInfo = null
+      } else {
+        viewInfo = await getFromDesignDoc()
+        migrate = !!viewInfo
+      }
     }
   }
+  if (migrate) {
+    await migrateToInMemoryView(db, viewName)
+  }
   if (!viewInfo) {
     throw "View does not exist."
   }
@@ -193,9 +212,6 @@ exports.fetchView = async ctx => {
   const db = new CouchDB(appId)
   const { calculation, group, field } = ctx.query
   const viewInfo = await getView(db, viewName)
-  if (!viewInfo) {
-    throw "View does not exist."
-  }
   let response
   if (env.SELF_HOSTED) {
     response = await db.query(`database/${viewName}`, {

packages/server/src/api/controllers/view/index.js

@@ -2,193 +2,93 @@ const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
 const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
+const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
-const {
-  ViewNames,
-  generateMemoryViewID,
-  getMemoryViewParams,
-} = require("../../../db/utils")
-const env = require("../../../environment")
 
-async function getView(db, viewName) {
+exports.fetch = async ctx => {
-  if (env.SELF_HOSTED) {
+  const db = new CouchDB(ctx.appId)
-    const designDoc = await db.get("_design/database")
+  ctx.body = await getViews(db)
-    return designDoc.views[viewName]
-  } else {
-    const viewDoc = await db.get(generateMemoryViewID(viewName))
-    return viewDoc.view
-  }
 }
 
-async function getViews(db) {
+exports.save = async ctx => {
-  const response = []
+  const db = new CouchDB(ctx.appId)
-  if (env.SELF_HOSTED) {
+  const { originalName, ...viewToSave } = ctx.request.body
-    const designDoc = await db.get("_design/database")
+  const view = viewTemplate(viewToSave)
-    for (let name of Object.keys(designDoc.views)) {
+
-      // Only return custom views, not built ins
+  if (!viewToSave.name) {
-      if (Object.values(ViewNames).indexOf(name) !== -1) {
+    ctx.throw(400, "Cannot create view without a name")
-        continue
-      }
-      response.push({
-        name,
-        ...designDoc.views[name],
-      })
-    }
-  } else {
-    const views = (
-      await db.allDocs(
-        getMemoryViewParams({
-          include_docs: true,
-        })
-      )
-    ).rows.map(row => row.doc)
-    for (let viewDoc of views) {
-      response.push({
-        name: viewDoc.name,
-        ...viewDoc.view,
-      })
-    }
   }
-  return response
-}
 
-async function saveView(db, originalName, viewToSave, viewTemplate) {
+  await saveView(db, originalName, viewToSave.name, view)
-  if (env.SELF_HOSTED) {
+
-    const designDoc = await db.get("_design/database")
+  // add views to table document
-    designDoc.views = {
+  const table = await db.get(ctx.request.body.tableId)
-      ...designDoc.views,
+  if (!table.views) table.views = {}
-      [viewToSave.name]: viewTemplate,
+  if (!view.meta.schema) {
-    }
+    view.meta.schema = table.schema
-    // view has been renamed
-    if (originalName) {
-      delete designDoc.views[originalName]
-    }
-    await db.put(designDoc)
-  } else {
-    const id = generateMemoryViewID(viewToSave.name)
-    const originalId = originalName ? generateMemoryViewID(originalName) : null
-    const viewDoc = {
-      _id: id,
-      view: viewTemplate,
-      name: viewToSave.name,
-      tableId: viewTemplate.meta.tableId,
-    }
-    try {
-      const old = await db.get(id)
-      if (originalId) {
-        const originalDoc = await db.get(originalId)
-        await db.remove(originalDoc._id, originalDoc._rev)
-      }
-      if (old && old._rev) {
-        viewDoc._rev = old._rev
-      }
-    } catch (err) {
-      // didn't exist, just skip
-    }
-    await db.put(viewDoc)
   }
-}
+  table.views[viewToSave.name] = view.meta
+  if (originalName) {
+    delete table.views[originalName]
+  }
+  await db.put(table)
 
-async function deleteView(db, viewName) {
+  ctx.body = {
-  if (env.SELF_HOSTED) {
+    ...table.views[viewToSave.name],
-    const designDoc = await db.get("_design/database")
+    name: viewToSave.name,
-    const view = designDoc.views[viewName]
-    delete designDoc.views[viewName]
-    await db.put(designDoc)
-    return view
-  } else {
-    const id = generateMemoryViewID(viewName)
-    const viewDoc = await db.get(id)
-    await db.remove(viewDoc._id, viewDoc._rev)
-    return viewDoc.view
   }
 }
 
-const controller = {
+exports.destroy = async ctx => {
-  fetch: async ctx => {
+  const db = new CouchDB(ctx.appId)
-    const db = new CouchDB(ctx.appId)
+  const viewName = decodeURI(ctx.params.viewName)
-    ctx.body = await getViews(db)
+  const view = await deleteView(db, viewName)
-  },
+  const table = await db.get(view.meta.tableId)
-  save: async ctx => {
+  delete table.views[viewName]
-    const db = new CouchDB(ctx.appId)
+  await db.put(table)
-    const { originalName, ...viewToSave } = ctx.request.body
-    const view = viewTemplate(viewToSave)
 
-    if (!viewToSave.name) {
+  ctx.body = view
-      ctx.throw(400, "Cannot create view without a name")
+}
-    }
-
-    await saveView(db, originalName, viewToSave, view)
-
-    // add views to table document
-    const table = await db.get(ctx.request.body.tableId)
-    if (!table.views) table.views = {}
-    if (!view.meta.schema) {
-      view.meta.schema = table.schema
-    }
-    table.views[viewToSave.name] = view.meta
-    if (originalName) {
-      delete table.views[originalName]
-    }
-    await db.put(table)
-
-    ctx.body = {
-      ...table.views[viewToSave.name],
-      name: viewToSave.name,
-    }
-  },
-  destroy: async ctx => {
-    const db = new CouchDB(ctx.appId)
-    const viewName = decodeURI(ctx.params.viewName)
-    const view = await deleteView(db, viewName)
-    const table = await db.get(view.meta.tableId)
-    delete table.views[viewName]
-    await db.put(table)
 
-    ctx.body = view
+exports.exportView = async ctx => {
-  },
+  const db = new CouchDB(ctx.appId)
-  exportView: async ctx => {
+  const viewName = decodeURI(ctx.query.view)
-    const db = new CouchDB(ctx.appId)
+  const view = await getView(db, viewName)
-    const viewName = decodeURI(ctx.query.view)
-    const view = await getView(db, viewName)
 
-    const format = ctx.query.format
+  const format = ctx.query.format
-    if (!format) {
+  if (!format) {
-      ctx.throw(400, "Format must be specified, either csv or json")
+    ctx.throw(400, "Format must be specified, either csv or json")
-    }
+  }
 
-    if (view) {
+  if (view) {
-      ctx.params.viewName = viewName
+    ctx.params.viewName = viewName
-      // Fetch view rows
+    // Fetch view rows
-      ctx.query = {
+    ctx.query = {
-        group: view.meta.groupBy,
+      group: view.meta.groupBy,
-        calculation: view.meta.calculation,
+      calculation: view.meta.calculation,
-        stats: !!view.meta.field,
+      stats: !!view.meta.field,
-        field: view.meta.field,
+      field: view.meta.field,
-      }
-    } else {
-      // table all_ view
-      /* istanbul ignore next */
-      ctx.params.viewName = viewName
     }
+  } else {
+    // table all_ view
+    /* istanbul ignore next */
+    ctx.params.viewName = viewName
+  }
 
-    await fetchView(ctx)
+  await fetchView(ctx)
 
-    let schema = view && view.meta && view.meta.schema
+  let schema = view && view.meta && view.meta.schema
-    if (!schema) {
+  if (!schema) {
-      const tableId = ctx.params.tableId || view.meta.tableId
+    const tableId = ctx.params.tableId || view.meta.tableId
-      const table = await db.get(tableId)
+    const table = await db.get(tableId)
-      schema = table.schema
+    schema = table.schema
-    }
+  }
 
-    // Export part
+  // Export part
-    let headers = Object.keys(schema)
+  let headers = Object.keys(schema)
-    const exporter = exporters[format]
+  const exporter = exporters[format]
-    const filename = `${viewName}.${format}`
+  const filename = `${viewName}.${format}`
-    // send down the file
+  // send down the file
-    ctx.attachment(filename)
+  ctx.attachment(filename)
-    ctx.body = apiFileReturn(exporter(headers, ctx.body))
+  ctx.body = apiFileReturn(exporter(headers, ctx.body))
-  },
 }
-
-module.exports = controller

packages/server/src/api/controllers/view/utils.js

@@ -0,0 +1,109 @@
+const {
+  ViewNames,
+  generateMemoryViewID,
+  getMemoryViewParams,
+} = require("../../../db/utils")
+const env = require("../../../environment")
+
+exports.getView = async (db, viewName) => {
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
+    return designDoc.views[viewName]
+  } else {
+    const viewDoc = await db.get(generateMemoryViewID(viewName))
+    return viewDoc.view
+  }
+}
+
+exports.getViews = async db => {
+  const response = []
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
+    for (let name of Object.keys(designDoc.views)) {
+      // Only return custom views, not built ins
+      if (Object.values(ViewNames).indexOf(name) !== -1) {
+        continue
+      }
+      response.push({
+        name,
+        ...designDoc.views[name],
+      })
+    }
+  } else {
+    const views = (
+      await db.allDocs(
+        getMemoryViewParams({
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+    for (let viewDoc of views) {
+      response.push({
+        name: viewDoc.name,
+        ...viewDoc.view,
+      })
+    }
+  }
+  return response
+}
+
+exports.saveView = async (db, originalName, viewName, viewTemplate) => {
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
+    designDoc.views = {
+      ...designDoc.views,
+      [viewName]: viewTemplate,
+    }
+    // view has been renamed
+    if (originalName) {
+      delete designDoc.views[originalName]
+    }
+    await db.put(designDoc)
+  } else {
+    const id = generateMemoryViewID(viewName)
+    const originalId = originalName ? generateMemoryViewID(originalName) : null
+    const viewDoc = {
+      _id: id,
+      view: viewTemplate,
+      name: viewName,
+      tableId: viewTemplate.meta.tableId,
+    }
+    try {
+      const old = await db.get(id)
+      if (originalId) {
+        const originalDoc = await db.get(originalId)
+        await db.remove(originalDoc._id, originalDoc._rev)
+      }
+      if (old && old._rev) {
+        viewDoc._rev = old._rev
+      }
+    } catch (err) {
+      // didn't exist, just skip
+    }
+    await db.put(viewDoc)
+  }
+}
+
+exports.deleteView = async (db, viewName) => {
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
+    const view = designDoc.views[viewName]
+    delete designDoc.views[viewName]
+    await db.put(designDoc)
+    return view
+  } else {
+    const id = generateMemoryViewID(viewName)
+    const viewDoc = await db.get(id)
+    await db.remove(viewDoc._id, viewDoc._rev)
+    return viewDoc.view
+  }
+}
+
+exports.migrateToInMemoryView = async (db, viewName) => {
+  // delete the view initially
+  const designDoc = await db.get("_design/database")
+  const view = designDoc.views[viewName]
+  delete designDoc.views[viewName]
+  await db.put(designDoc)
+  await exports.saveView(db, null, viewName, view)
+}

packages/server/src/api/routes/tests/view.spec.js

@@ -205,7 +205,7 @@ describe("/views", () => {
   })
 
   describe("exportView", () => {
-    it("should be able to delete a view", async () => {
+    it("should be able to export a view", async () => {
       await config.createTable(priceTable())
       await config.createRow()
       const view = await config.createView()

packages/server/src/environment.js

@@ -26,7 +26,7 @@ module.exports = {
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   MINIO_URL: process.env.MINIO_URL,
   WORKER_URL: process.env.WORKER_URL,
-  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
+  SELF_HOSTED: process.env.SELF_HOSTED,
   AWS_REGION: process.env.AWS_REGION,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,