Merge pull request #3989 from Budibase/fix/invalidate-variables-on-deletion

Invalidate dynamic variables when they are removed from datasource

packages/server/src/api/controllers/datasource.js

@@ -10,6 +10,7 @@ const {
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
 const { integrations } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
+const { invalidateDynamicVariables } = require("../../threads/utils")
 
 exports.fetch = async function (ctx) {
   const database = new CouchDB(ctx.appId)
@@ -57,10 +58,43 @@ exports.buildSchemaFromDb = async function (ctx) {
   ctx.body = response
 }
 
+/**
+ * Check for variables that have been updated or removed and invalidate them.
+ */
+const invalidateVariables = async (existingDatasource, updatedDatasource) => {
+  const existingVariables = existingDatasource.config.dynamicVariables
+  const updatedVariables = updatedDatasource.config.dynamicVariables
+  const toInvalidate = []
+
+  if (!existingVariables) {
+    return
+  }
+
+  if (!updatedVariables) {
+    // invalidate all
+    toInvalidate.push(...existingVariables)
+  } else {
+    // invaldate changed / removed
+    existingVariables.forEach(existing => {
+      const unchanged = updatedVariables.find(
+        updated =>
+          existing.name === updated.name &&
+          existing.queryId === updated.queryId &&
+          existing.value === updated.value
+      )
+      if (!unchanged) {
+        toInvalidate.push(existing)
+      }
+    })
+  }
+  await invalidateDynamicVariables(toInvalidate)
+}
+
 exports.update = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasourceId = ctx.params.datasourceId
   let datasource = await db.get(datasourceId)
+  await invalidateVariables(datasource, ctx.request.body)
   datasource = { ...datasource, ...ctx.request.body }
 
   const response = await db.put(datasource)

packages/server/src/api/routes/tests/datasource.spec.js

@@ -4,6 +4,7 @@ let setup = require("./utilities")
 let { basicDatasource } = setup.structures
 let { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const pg = require("pg")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 
 describe("/datasources", () => {
   let request = setup.getRequest()
@@ -31,6 +32,50 @@ describe("/datasources", () => {
     })
   })
 
+  describe("update", () => {
+    it("should update an existing datasource", async () => {
+      datasource.name = "Updated Test"
+      const res = await request
+        .put(`/api/datasources/${datasource._id}`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect('Content-Type', /json/)
+        .expect(200)
+
+      expect(res.body.datasource.name).toEqual("Updated Test")
+      expect(res.body.errors).toBeUndefined()
+    })
+
+    describe("dynamic variables", () => {
+      async function preview(datasource, fields) {
+        return config.previewQuery(request, config, datasource, fields)
+      }
+
+      it("should invalidate changed or removed variables", async () => {
+        const { datasource, query } = await config.dynamicVariableDatasource()
+        // preview once to cache variables
+        await preview(datasource, { path: "www.test.com", queryString: "test={{ variable3 }}" })
+        // check variables in cache
+        let contents = await checkCacheForDynamicVariable(query._id, "variable3")
+        expect(contents.rows.length).toEqual(1)
+        
+        // update the datasource to remove the variables
+        datasource.config.dynamicVariables = []
+        const res = await request
+          .put(`/api/datasources/${datasource._id}`)
+          .send(datasource)
+          .set(config.defaultHeaders())
+          .expect('Content-Type', /json/)
+          .expect(200)
+        expect(res.body.errors).toBeUndefined()
+
+        // check variables no longer in cache
+        contents = await checkCacheForDynamicVariable(query._id, "variable3")
+        expect(contents).toBe(null)
+      })
+    })
+  })
+
   describe("fetch", () => {
     it("returns all the datasources from the server", async () => {
       const res = await request

packages/server/src/api/routes/tests/query.spec.js

@@ -229,52 +229,14 @@ describe("/queries", () => {
     })
   })
 
-  describe("test variables", () => {
-    async function restDatasource(cfg) {
-      return await config.createDatasource({
-        datasource: {
-          ...basicDatasource().datasource,
-          source: "REST",
-          config: cfg || {},
-        },
-      })
-    }
-
-    async function dynamicVariableDatasource() {
-      const datasource = await restDatasource()
-      const basedOnQuery = await config.createQuery({
-        ...basicQuery(datasource._id),
-        fields: {
-          path: "www.google.com",
-        },
-      })
-      await config.updateDatasource({
-        ...datasource,
-        config: {
-          dynamicVariables: [
-            { queryId: basedOnQuery._id, name: "variable3", value: "{{ data.0.[value] }}" }
-          ]
-        }
-      })
-      return { datasource, query: basedOnQuery }
-    }
+  describe("variables", () => {
 
     async function preview(datasource, fields) {
-      return await request
-        .post(`/api/queries/preview`)
-        .send({
-          datasourceId: datasource._id,
-          parameters: {},
-          fields,
-          queryVerb: "read",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      return config.previewQuery(request, config, datasource, fields)
     }
 
     it("should work with static variables", async () => {
-      const datasource = await restDatasource({
+      const datasource = await config.restDatasource({
         staticVariables: {
           variable: "google",
           variable2: "1",
@@ -290,7 +252,7 @@ describe("/queries", () => {
     })
 
     it("should work with dynamic variables", async () => {
-      const { datasource } = await dynamicVariableDatasource()
+      const { datasource } = await config.dynamicVariableDatasource()
       const res = await preview(datasource, {
         path: "www.google.com",
         queryString: "test={{ variable3 }}",
@@ -300,7 +262,7 @@ describe("/queries", () => {
     })
 
     it("check that it automatically retries on fail with cached dynamics", async () => {
-      const { datasource, query: base } = await dynamicVariableDatasource()
+      const { datasource, query: base } = await config.dynamicVariableDatasource()
       // preview once to cache
       await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
       // check its in cache
@@ -313,5 +275,24 @@ describe("/queries", () => {
       expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
       expect(res.body.rows[0].fails).toEqual(1)
     })
+
+    it("deletes variables when linked query is deleted", async () => {
+      const { datasource, query: base } = await config.dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      let contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+
+      // delete the query
+      await request
+        .delete(`/api/queries/${base._id}/${base._rev}`)
+        .set(config.defaultHeaders())
+        .expect(200)
+
+      // check variables no longer in cache
+      contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents).toBe(null)
+    })
   })
 })

packages/server/src/tests/utilities/TestConfiguration.js

@@ -326,6 +326,53 @@ class TestConfiguration {
     return this.datasource
   }
 
+  async restDatasource(cfg) {
+    return this.createDatasource({
+      datasource: {
+        ...basicDatasource().datasource,
+        source: "REST",
+        config: cfg || {},
+      },
+    })
+  }
+
+  async dynamicVariableDatasource() {
+    let datasource = await this.restDatasource()
+    const basedOnQuery = await this.createQuery({
+      ...basicQuery(datasource._id),
+      fields: {
+        path: "www.google.com",
+      },
+    })
+    datasource = await this.updateDatasource({
+      ...datasource,
+      config: {
+        dynamicVariables: [
+          {
+            queryId: basedOnQuery._id,
+            name: "variable3",
+            value: "{{ data.0.[value] }}",
+          },
+        ],
+      },
+    })
+    return { datasource, query: basedOnQuery }
+  }
+
+  async previewQuery(request, config, datasource, fields) {
+    return request
+      .post(`/api/queries/preview`)
+      .send({
+        datasourceId: datasource._id,
+        parameters: {},
+        fields,
+        queryVerb: "read",
+      })
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+  }
+
   async createQuery(config = null) {
     if (!this.datasource && !config) {
       throw "No data source created for query."