Merge pull request #3932 from Budibase/feature/query-variables

Fixes for new rest datasource
4 years ago · 8bf0c86c92
15 changed files with 79 additions and 39 deletions
--- a/packages/bbui/src/Tabs/Tabs.svelte
+++ b/packages/bbui/src/Tabs/Tabs.svelte
@ -10,6 +10,8 @@
  export let noHorizPadding = false
  export let quiet = false
  export let emphasized = false
+  // overlay content from the tab bar onto tabs e.g. for a dropdown
+  export let onTop = false

  let thisSelected = undefined

@ -78,6 +80,7 @@
    'spectrum-Tabs--quiet'} spectrum-Tabs--{vertical
    ? 'vertical'
    : 'horizontal'}"
+  class:onTop
 >
  <slot />
  {#if $tab.info}
@ -98,7 +101,9 @@
  .quiet {
    border-bottom: none !important;
  }
-
+  .onTop {
+    z-index: 20;
+  }
  .spectrum-Tabs {
    padding-left: var(--spacing-xl);
    padding-right: var(--spacing-xl);
--- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte
@ -137,7 +137,7 @@
            selected={$queries.selected === query._id}
            on:click={() => onClickQuery(query)}
          >
-            <EditQueryPopover {query} />
+            <EditQueryPopover {query} {onClickQuery} />
          </NavItem>
        {/each}
      {/if}
--- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationBuilder.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationBuilder.svelte
@ -58,7 +58,7 @@
    />
  {/if}
  <div>
-    <ActionButton on:click={() => openConfigModal()} con="Add"
+    <ActionButton on:click={() => openConfigModal()} icon="Add"
      >Add authentication</ActionButton
    >
  </div>
--- a/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditQueryPopover.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditQueryPopover.svelte
@ -5,22 +5,29 @@
  import { datasources, queries } from "stores/backend"

  export let query
+  export let onClickQuery

  let confirmDeleteDialog

  async function deleteQuery() {
    const wasSelectedQuery = $queries.selected
-    const selectedDatasource = $datasources.selected
+    // need to calculate this before the query is deleted
+    const navigateToDatasource = wasSelectedQuery === query._id
+
    await queries.delete(query)
-    if (wasSelectedQuery === query._id) {
-      $goto(`./datasource/${selectedDatasource}`)
+    await datasources.fetch()
+
+    if (navigateToDatasource) {
+      await datasources.select(query.datasourceId)
+      $goto(`./datasource/${query.datasourceId}`)
    }
    notifications.success("Query deleted")
  }

  async function duplicateQuery() {
    try {
-      await queries.duplicate(query)
+      const newQuery = await queries.duplicate(query)
+      onClickQuery(newQuery)
    } catch (e) {
      notifications.error(e.message)
    }
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
@ -232,8 +232,12 @@
    const datasourceUrl = datasource?.config.url
    const qs = query?.fields.queryString
    breakQs = restUtils.breakQueryString(qs)
-    if (datasourceUrl && !query.fields.path?.startsWith(datasourceUrl)) {
-      const path = query.fields.path
+    const path = query.fields.path
+    if (
+      datasourceUrl &&
+      !path?.startsWith("http") &&
+      !path?.startsWith("{{") // don't substitute the datasource url when query starts with a variable e.g. the upgrade path
+    ) {
      query.fields.path = `${datasource.config.url}/${path ? path : ""}`
    }
    url = buildUrl(query.fields.path, breakQs)
@ -306,7 +310,7 @@
          </div>
          <Button cta disabled={!url} on:click={runQuery}>Send</Button>
        </div>
-        <Tabs selected="Bindings" quiet noPadding noHorizPadding>
+        <Tabs selected="Bindings" quiet noPadding noHorizPadding onTop>
          <Tab title="Bindings">
            <KeyValueBuilder
              bind:object={bindings}
@ -450,7 +454,7 @@
                <Layout noPadding gap="S">
                  <Body size="S">
                    Create dynamic variables based on response body or headers
-                    from other queries.
+                    from this query.
                  </Body>
                  <KeyValueBuilder
                    bind:object={dynamicVariables}
--- a/packages/builder/src/stores/backend/queries.js
+++ b/packages/builder/src/stores/backend/queries.js
@ -134,7 +134,7 @@ export function createQueriesStore() {
        list.map(q => q.name)
      )

-      actions.save(datasourceId, newQuery)
+      return actions.save(datasourceId, newQuery)
    },
  }

--- a/packages/server/src/api/controllers/query/import/sources/base/index.ts
+++ b/packages/server/src/api/controllers/query/import/sources/base/index.ts
@ -1,7 +1,7 @@
 import { Query, QueryParameter } from "../../../../../../definitions/datasource"
+import { URL } from "url"

 export interface ImportInfo {
-  url: string
  name: string
 }

@ -23,6 +23,7 @@ export abstract class ImportSource {
    name: string,
    method: string,
    path: string,
+    url: URL,
    queryString: string,
    headers: object = {},
    parameters: QueryParameter[] = [],
@ -33,6 +34,7 @@ export abstract class ImportSource {
    const transformer = "return data"
    const schema = {}
    path = this.processPath(path)
+    path = `${url.origin}/${path}`
    queryString = this.processQuery(queryString)
    const requestBody = JSON.stringify(body, null, 2)

--- a/packages/server/src/api/controllers/query/import/sources/curl.ts
+++ b/packages/server/src/api/controllers/query/import/sources/curl.ts
@ -60,16 +60,19 @@ export class Curl extends ImportSource {
    return true
  }

+  getUrl = (): URL => {
+    return new URL(this.curl.raw_url)
+  }
+
  getInfo = async (): Promise<ImportInfo> => {
-    const url = new URL(this.curl.url)
+    const url = this.getUrl()
    return {
-      url: url.origin,
      name: url.hostname,
    }
  }

  getQueries = async (datasourceId: string): Promise<Query[]> => {
-    const url = new URL(this.curl.raw_url)
+    const url = this.getUrl()
    const name = url.pathname
    const path = url.pathname
    const method = this.curl.method
@ -87,6 +90,7 @@ export class Curl extends ImportSource {
      name,
      method,
      path,
+      url,
      queryString,
      headers,
      [],
--- a/packages/server/src/api/controllers/query/import/sources/openapi2.ts
+++ b/packages/server/src/api/controllers/query/import/sources/openapi2.ts
@ -2,6 +2,7 @@ import { ImportInfo } from "./base"
 import { Query, QueryParameter } from "../../../../../definitions/datasource"
 import { OpenAPIV2 } from "openapi-types"
 import { OpenAPISource } from "./base/openapi"
+import { URL } from "url"

 const parameterNotRef = (
  param: OpenAPIV2.Parameter | OpenAPIV2.ReferenceObject
@ -55,20 +56,22 @@ export class OpenAPI2 extends OpenAPISource {
    }
  }

-  getInfo = async (): Promise<ImportInfo> => {
+  getUrl = (): URL => {
    const scheme = this.document.schemes?.includes("https") ? "https" : "http"
    const basePath = this.document.basePath || ""
    const host = this.document.host || "<host>"
-    const url = `${scheme}://${host}${basePath}`
-    const name = this.document.info.title || "Swagger Import"
+    return new URL(`${scheme}://${host}${basePath}`)
+  }

+  getInfo = async (): Promise<ImportInfo> => {
+    const name = this.document.info.title || "Swagger Import"
    return {
-      url: url,
-      name: name,
+      name,
    }
  }

  getQueries = async (datasourceId: string): Promise<Query[]> => {
+    const url = this.getUrl()
    const queries = []

    for (let [path, pathItem] of Object.entries(this.document.paths)) {
@ -145,6 +148,7 @@ export class OpenAPI2 extends OpenAPISource {
          name,
          methodName,
          path,
+          url,
          queryString,
          headers,
          parameters,
--- a/packages/server/src/api/controllers/query/import/sources/tests/curl/curl.spec.js
+++ b/packages/server/src/api/controllers/query/import/sources/tests/curl/curl.spec.js
@ -35,7 +35,6 @@ describe("Curl Import", () => {
  it("returns import info", async () => {
    await init("get")
    const info = await curl.getInfo()
-    expect(info.url).toBe("http://example.com")
    expect(info.name).toBe("example.com")
  })
  
@ -67,8 +66,8 @@ describe("Curl Import", () => {
    }

    it("populates path", async () => {
-        await testPath("get", "")
-        await testPath("path", "paths/abc")
+        await testPath("get", "http://example.com/")
+        await testPath("path", "http://example.com/paths/abc")
    })

    const testHeaders = async (file, headers) => {
--- a/packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js
+++ b/packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js
@ -41,7 +41,6 @@ describe("OpenAPI2 Import", () => {
  const testImportInfo = async (file, extension) => {
    await init(file, extension)
    const info = await openapi2.getInfo()
-    expect(info.url).toBe("https://petstore.swagger.io/v2")
    expect(info.name).toBe("Swagger Petstore")
  }

@ -92,12 +91,12 @@ describe("OpenAPI2 Import", () => {

    it("populates path", async () => {
      const assertions = {
-        "createEntity" : "entities",
-        "getEntities" : "entities",
-        "getEntity" : "entities/{{entityId}}",
-        "updateEntity" : "entities/{{entityId}}",
-        "patchEntity" : "entities/{{entityId}}",
-        "deleteEntity" : "entities/{{entityId}}"
+        "createEntity" : "http://example.com/entities",
+        "getEntities" : "http://example.com/entities",
+        "getEntity" : "http://example.com/entities/{{entityId}}",
+        "updateEntity" : "http://example.com/entities/{{entityId}}",
+        "patchEntity" : "http://example.com/entities/{{entityId}}",
+        "deleteEntity" : "http://example.com/entities/{{entityId}}"
      }
      await runTests("crud", testPath, assertions)
    })
--- a/packages/server/src/api/controllers/query/import/tests/index.spec.js
+++ b/packages/server/src/api/controllers/query/import/tests/index.spec.js
@ -51,30 +51,24 @@ describe("Rest Importer", () => {
    await init(data)
    const info = await restImporter.getInfo()
    expect(info.name).toBe(assertions[key].name)
-    expect(info.url).toBe(assertions[key].url)
  }

  it("gets info", async () => {
    const assertions = {
      "oapi2CrudJson" : {
        name: "CRUD",
-        url: "http://example.com"
      },
      "oapi2CrudYaml" : {
        name: "CRUD",
-        url: "http://example.com"
      },
      "oapi2PetstoreJson" : {
        name: "Swagger Petstore",
-        url: "https://petstore.swagger.io/v2"
      },
      "oapi2PetstoreYaml" :{
        name: "Swagger Petstore",
-        url: "https://petstore.swagger.io/v2"
      },
      "curl": {
        name: "example.com",
-        url: "http://example.com"
      }
    }
    await runTest(testGetInfo, assertions)
--- a/packages/server/src/api/controllers/query/index.js
+++ b/packages/server/src/api/controllers/query/index.js
@ -8,6 +8,7 @@ const { BaseQueryVerbs } = require("../../../constants")
 const { Thread, ThreadType } = require("../../../threads")
 const { save: saveDatasource } = require("../datasource")
 const { RestImporter } = require("./import")
+const { invalidateDynamicVariables } = require("../../../threads/utils")

 const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })

@ -166,8 +167,28 @@ exports.executeV2 = async function (ctx) {
  return execute(ctx, { rowsOnly: false })
 }

+const removeDynamicVariables = async (db, queryId) => {
+  const query = await db.get(queryId)
+  const datasource = await db.get(query.datasourceId)
+  const dynamicVariables = datasource.config.dynamicVariables
+
+  if (dynamicVariables) {
+    // delete dynamic variables from the datasource
+    const newVariables = dynamicVariables.filter(dv => dv.queryId !== queryId)
+    datasource.config.dynamicVariables = newVariables
+    await db.put(datasource)
+
+    // invalidate the deleted variables
+    const variablesToDelete = dynamicVariables.filter(
+      dv => dv.queryId === queryId
+    )
+    await invalidateDynamicVariables(variablesToDelete)
+  }
+}
+
 exports.destroy = async function (ctx) {
  const db = new CouchDB(ctx.appId)
+  await removeDynamicVariables(db, ctx.params.queryId)
  await db.remove(ctx.params.queryId, ctx.params.revId)
  ctx.message = `Query deleted.`
  ctx.status = 200
--- a/packages/server/src/integrations/rest.ts
+++ b/packages/server/src/integrations/rest.ts
@ -171,7 +171,7 @@ module RestModule {
    getUrl(path: string, queryString: string): string {
      const main = `${path}?${queryString}`
      let complete = main
-      if (this.config.url && !main.startsWith(this.config.url)) {
+      if (this.config.url && !main.startsWith("http")) {
        complete = !this.config.url ? main : `${this.config.url}/${main}`
      }
      if (!complete.startsWith("http")) {
--- a/packages/server/src/threads/utils.js
+++ b/packages/server/src/threads/utils.js
@ -42,10 +42,11 @@ exports.checkCacheForDynamicVariable = async (queryId, variable) => {
 }

 exports.invalidateDynamicVariables = async cachedVars => {
+  const cache = await getClient()
  let promises = []
  for (let variable of cachedVars) {
    promises.push(
-      client.delete(makeVariableKey(variable.queryId, variable.name))
+      cache.delete(makeVariableKey(variable.queryId, variable.name))
    )
  }
  await Promise.all(promises)