Merge pull request #5881 from Budibase/fix/5850

Fixing issues with calculation views when self hosting and using real CouchDB views
4 years ago · 6c7fd0d7b6
7 changed files with 138 additions and 38 deletions
--- a/packages/backend-core/src/middleware/passport/tests/oidc.spec.js
+++ b/packages/backend-core/src/middleware/passport/tests/oidc.spec.js
@ -71,7 +71,7 @@ describe("oidc", () => {
  
  describe("authenticate", () => {    
    afterEach(() => {
-      jest.clearAllMocks();
+      jest.clearAllMocks()
    });

    // mock third party common authentication
@ -80,10 +80,10 @@ describe("oidc", () => {
    
    // mock the passport callback
    const mockDone = jest.fn()
+    const mockSaveUserFn = jest.fn()

    async function doAuthenticate() {
      const oidc = require("../oidc")
-      const mockSaveUserFn = jest.fn()
      const authenticate = await oidc.buildVerifyFn(mockSaveUserFn)

      await authenticate(
@ -105,11 +105,13 @@ describe("oidc", () => {
      expect(authenticateThirdParty).toHaveBeenCalledWith(
        user,
        false, 
-        mockDone)
+        mockDone,
+        mockSaveUserFn,
+      )
    }

    it("delegates authentication to third party common", async () => {
-      doTest()
+      await doTest()
    })

    it("uses JWT email to get email", async () => {
@ -118,7 +120,7 @@ describe("oidc", () => {
        email : "mock@budibase.com"
      }

-      doTest()
+      await doTest()
    })
  
    it("uses JWT username to get email", async () => {
@ -127,7 +129,7 @@ describe("oidc", () => {
        preferred_username : "mock@budibase.com"
      }

-      doTest()
+      await doTest()
    })

    it("uses JWT invalid username to get email", async () => {
--- a/packages/server/src/api/controllers/view/tests/snapshots/viewBuilder.spec.js.snap
+++ b/packages/server/src/api/controllers/view/tests/snapshots/viewBuilder.spec.js.snap
@ -1,9 +1,66 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP

+exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
+Object {
+  "map": "function (doc) {
+      if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
+      doc[\\"myField\\"] === undefined ||
+      doc[\\"myField\\"] === null ||
+      doc[\\"myField\\"] === \\"\\" ||
+      (Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
+  )) && (doc[\\"age\\"] > 17)) {
+        emit(doc[\\"_id\\"], doc[\\"myField\\"]);
+      }
+    }",
+  "meta": Object {
+    "calculation": "stats",
+    "field": "myField",
+    "filters": Array [
+      Object {
+        "condition": "MT",
+        "key": "age",
+        "value": 17,
+      },
+    ],
+    "groupBy": undefined,
+    "schema": Object {
+      "avg": Object {
+        "type": "number",
+      },
+      "count": Object {
+        "type": "number",
+      },
+      "field": Object {
+        "type": "string",
+      },
+      "max": Object {
+        "type": "number",
+      },
+      "min": Object {
+        "type": "number",
+      },
+      "sum": Object {
+        "type": "number",
+      },
+      "sumsqr": Object {
+        "type": "number",
+      },
+    },
+    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
+  },
+  "reduce": "_stats",
+}
+`;
+
 exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
 Object {
  "map": "function (doc) {
-      if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" ) {
+      if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
+      doc[\\"myField\\"] === undefined ||
+      doc[\\"myField\\"] === null ||
+      doc[\\"myField\\"] === \\"\\" ||
+      (Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
+  )) ) {
        emit(doc[\\"_id\\"], doc[\\"myField\\"]);
      }
    }",
--- a/packages/server/src/api/controllers/view/tests/viewBuilder.spec.js
+++ b/packages/server/src/api/controllers/view/tests/viewBuilder.spec.js
@ -44,4 +44,22 @@ describe("viewBuilder", () => {
      })).toMatchSnapshot()
    })
  })
+
+  describe("Calculate and filter", () => {
+    it("creates a view with the calculation statistics and filter schema", () => {
+      expect(viewTemplate({
+        "name": "Calculate View",
+        "field": "myField",
+        "calculation": "stats",
+        "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
+        "filters": [
+          {
+            "value": 17,
+            "condition": "MT",
+            "key": "age",
+          }
+        ]
+      })).toMatchSnapshot()
+    })
+  })
 });
--- a/packages/server/src/api/controllers/view/utils.js
+++ b/packages/server/src/api/controllers/view/utils.js
@ -7,6 +7,7 @@ const {
 } = require("../../../db/utils")
 const env = require("../../../environment")
 const { getAppDB } = require("@budibase/backend-core/context")
+const viewBuilder = require("./viewBuilder")

 exports.getView = async viewName => {
  const db = getAppDB()
@ -114,7 +115,8 @@ exports.deleteView = async viewName => {
 exports.migrateToInMemoryView = async (db, viewName) => {
  // delete the view initially
  const designDoc = await db.get("_design/database")
-  const view = designDoc.views[viewName]
+  // run the view back through the view builder to update it
+  const view = viewBuilder(designDoc.views[viewName].meta)
  delete designDoc.views[viewName]
  await db.put(designDoc)
  await exports.saveView(db, null, viewName, view)
@ -123,7 +125,7 @@ exports.migrateToInMemoryView = async (db, viewName) => {
 exports.migrateToDesignView = async (db, viewName) => {
  let view = await db.get(generateMemoryViewID(viewName))
  const designDoc = await db.get("_design/database")
-  designDoc.views[viewName] = view.view
+  designDoc.views[viewName] = viewBuilder(view.view.meta)
  await db.put(designDoc)
  await db.remove(view._id, view._rev)
 }
--- a/packages/server/src/api/controllers/view/viewBuilder.js
+++ b/packages/server/src/api/controllers/view/viewBuilder.js
@ -10,6 +10,12 @@ const TOKEN_MAP = {
  OR: "||",
 }

+const CONDITIONS = {
+  EMPTY: "EMPTY",
+  NOT_EMPTY: "NOT_EMPTY",
+  CONTAINS: "CONTAINS",
+}
+
 const isEmptyExpression = key => {
  return `(
      doc["${key}"] === undefined ||
@ -77,13 +83,13 @@ function parseFilterExpression(filters) {
      expression.push(TOKEN_MAP[filter.conjunction])
    }

-    if (filter.condition === "CONTAINS") {
+    if (filter.condition === CONDITIONS.CONTAINS) {
      expression.push(
        `doc["${filter.key}"].${TOKEN_MAP[filter.condition]}("${filter.value}")`
      )
-    } else if (filter.condition === "EMPTY") {
+    } else if (filter.condition === CONDITIONS.EMPTY) {
      expression.push(isEmptyExpression(filter.key))
-    } else if (filter.condition === "NOT_EMPTY") {
+    } else if (filter.condition === CONDITIONS.NOT_EMPTY) {
      expression.push(`!${isEmptyExpression(filter.key)}`)
    } else {
      const value =
@ -125,22 +131,37 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
  if (filters && filters.length > 0 && filters[0].conjunction) {
    delete filters[0].conjunction
  }
-  const parsedFilters = parseFilterExpression(filters)
-  const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
-
-  const emitExpression = parseEmitExpression(field, groupBy)
-
-  const reduction = field && calculation ? { reduce: `_${calculation}` } : {}

-  let schema = null
+  let schema = null,
+    statFilter = null

  if (calculation) {
    schema = {
      ...(groupBy ? GROUP_PROPERTY : FIELD_PROPERTY),
      ...SCHEMA_MAP[calculation],
    }
+    if (
+      !filters.find(
+        filter =>
+          filter.key === field && filter.condition === CONDITIONS.NOT_EMPTY
+      )
+    ) {
+      statFilter = parseFilterExpression([
+        { key: field, condition: CONDITIONS.NOT_EMPTY },
+      ])
+    }
  }

+  const parsedFilters = parseFilterExpression(filters)
+  const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
+
+  const emitExpression = parseEmitExpression(field, groupBy)
+  const tableExpression = `doc.tableId === "${tableId}"`
+  const coreExpression = statFilter
+    ? `(${tableExpression} && ${statFilter})`
+    : tableExpression
+  const reduction = field && calculation ? { reduce: `_${calculation}` } : {}
+
  return {
    meta: {
      field,
@ -151,7 +172,7 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
      calculation,
    },
    map: `function (doc) {
-      if (doc.tableId === "${tableId}" ${filterExpression}) {
+      if (${coreExpression} ${filterExpression}) {
        ${emitExpression}
      }
    }`,
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@ -1014,10 +1014,10 @@
  resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
  integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==

-"@budibase/backend-core@1.0.154":
-  version "1.0.154"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.154.tgz#c310834892e7621778b07579464955487c5c9830"
-  integrity sha512-mcZxt8XhGgOB4XRHKkWTvBEI4HGp2bo8qyzOJRCvDqlg56S9zqGJDl75Z0N/Wc8N3I53QRcxISerj48odX172A==
+"@budibase/backend-core@1.0.160":
+  version "1.0.160"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.160.tgz#000e3b5a3ed91e73a542b4caa202a6f147d91294"
+  integrity sha512-XfAFU6sRPrCSEKlm58WeuPw8lUoJK+KwO0tcbT+bB2Nb7XCHplskryEgk/PM9ujRU6SMPDx11zKeqRebHlycbA==
  dependencies:
    "@techpass/passport-openidconnect" "^0.3.0"
    aws-sdk "^2.901.0"
@ -1091,12 +1091,12 @@
    svelte-flatpickr "^3.2.3"
    svelte-portal "^1.0.0"

-"@budibase/pro@1.0.154":
-  version "1.0.154"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.154.tgz#f4e31e30376b206159b711224038141d73a1118e"
-  integrity sha512-+O6bemrcgyWG4a+D5dIOoZ+LGjW4aN7tRdFeZqoaIPCc1pA6zNtLUkM1nb+Laafuwq2Aht37vEuaRy7jfzVprA==
+"@budibase/pro@1.0.160":
+  version "1.0.160"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.160.tgz#921c4e3f65b866d84292644dfd8793c4d0b667c7"
+  integrity sha512-p+Jhnk1n98CWCJXydSQSO7a+HDpqGAHekGQbOR7aayuwuoYzyOXxTcHNLdBp+3lkXhLSZq9oIwfEGpgdrrhXPA==
  dependencies:
-    "@budibase/backend-core" "1.0.154"
+    "@budibase/backend-core" "1.0.160"
    node-fetch "^2.6.1"

 "@budibase/standard-components@^0.9.139":
--- a/packages/worker/yarn.lock
+++ b/packages/worker/yarn.lock
@ -293,10 +293,10 @@
  resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
  integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==

-"@budibase/backend-core@1.0.154":
-  version "1.0.154"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.154.tgz#c310834892e7621778b07579464955487c5c9830"
-  integrity sha512-mcZxt8XhGgOB4XRHKkWTvBEI4HGp2bo8qyzOJRCvDqlg56S9zqGJDl75Z0N/Wc8N3I53QRcxISerj48odX172A==
+"@budibase/backend-core@1.0.160":
+  version "1.0.160"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.160.tgz#000e3b5a3ed91e73a542b4caa202a6f147d91294"
+  integrity sha512-XfAFU6sRPrCSEKlm58WeuPw8lUoJK+KwO0tcbT+bB2Nb7XCHplskryEgk/PM9ujRU6SMPDx11zKeqRebHlycbA==
  dependencies:
    "@techpass/passport-openidconnect" "^0.3.0"
    aws-sdk "^2.901.0"
@ -321,12 +321,12 @@
    uuid "^8.3.2"
    zlib "^1.0.5"

-"@budibase/pro@1.0.154":
-  version "1.0.154"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.154.tgz#f4e31e30376b206159b711224038141d73a1118e"
-  integrity sha512-+O6bemrcgyWG4a+D5dIOoZ+LGjW4aN7tRdFeZqoaIPCc1pA6zNtLUkM1nb+Laafuwq2Aht37vEuaRy7jfzVprA==
+"@budibase/pro@1.0.160":
+  version "1.0.160"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.160.tgz#921c4e3f65b866d84292644dfd8793c4d0b667c7"
+  integrity sha512-p+Jhnk1n98CWCJXydSQSO7a+HDpqGAHekGQbOR7aayuwuoYzyOXxTcHNLdBp+3lkXhLSZq9oIwfEGpgdrrhXPA==
  dependencies:
-    "@budibase/backend-core" "1.0.154"
+    "@budibase/backend-core" "1.0.160"
    node-fetch "^2.6.1"

 "@cspotcode/source-map-consumer@0.8.0":