WIP: Rest API import

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -6,6 +6,8 @@
   import { generateID } from "../../utils/helpers"
   import Icon from "../../Icon/Icon.svelte"
   import Link from "../../Link/Link.svelte"
+  import Tag from "../../Tags/Tag.svelte"
+  import Tags from "../../Tags/Tags.svelte"
 
   const BYTES_IN_KB = 1000
   const BYTES_IN_MB = 1000000
@@ -18,6 +20,7 @@
   export let handleFileTooLarge = null
   export let gallery = true
   export let error = null
+  export let fileTags = []
 
   const dispatch = createEventDispatcher()
   const imageExtensions = [
@@ -278,6 +281,19 @@
           <br />
           from your computer
         </p>
+        {#if fileTags.length}
+          <Tags>
+            <div class="tags">
+              {#each fileTags as tag}
+                <div class="tag">
+                  <Tag>
+                    {tag}
+                  </Tag>
+                </div>
+              {/each}
+            </div>
+          </Tags>
+        {/if}
       {/if}
     </div>
   </div>
@@ -390,4 +406,15 @@
   .disabled .spectrum-Heading--sizeL {
     color: var(--spectrum-alias-text-color-disabled);
   }
+
+  .tags {
+    margin-top: 20px;
+    display: flex;
+    flex-wrap: wrap;
+    justify-content: center;
+  }
+
+  .tag {
+    margin-top: 8px;
+  }
 </style>

packages/bbui/src/Form/Dropzone.svelte

@@ -12,6 +12,7 @@
   export let processFiles = undefined
   export let handleFileTooLarge = undefined
   export let gallery = true
+  export let fileTags = []
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -29,6 +30,7 @@
     {processFiles}
     {handleFileTooLarge}
     {gallery}
+    {fileTags}
     on:change={onChange}
   />
 </Field>

packages/bbui/src/Modal/ModalContent.svelte

@@ -18,10 +18,22 @@
   export let disabled = false
   export let showDivider = true
 
+  export let showSecondaryButton = false
+  export let secondaryButtonText = undefined
+  export let secondaryAction = undefined
+
   const { hide, cancel } = getContext(Context.Modal)
   let loading = false
   $: confirmDisabled = disabled || loading
 
+  async function secondary() {
+    loading = true
+    if (!secondaryAction || (await secondaryAction()) !== false) {
+      hide()
+    }
+    loading = false
+  }
+
   async function confirm() {
     loading = true
     if (!onConfirm || (await onConfirm()) !== false) {
@@ -73,6 +85,15 @@
         class="spectrum-ButtonGroup spectrum-Dialog-buttonGroup spectrum-Dialog-buttonGroup--noFooter"
       >
         <slot name="footer" />
+
+        {#if showSecondaryButton && secondaryButtonText && secondaryAction}
+          <div class="secondary-action">
+            <Button group secondary on:click={secondary}
+              >{secondaryButtonText}</Button
+            >
+          </div>
+        {/if}
+
         {#if showCancelButton}
           <Button group secondary on:click={close}>{cancelText}</Button>
         {/if}
@@ -136,4 +157,8 @@
     display: flex;
     justify-content: space-between;
   }
+
+  .secondary-action {
+    margin-right: auto;
+  }
 </style>

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -6,12 +6,18 @@
   import { IntegrationNames } from "constants"
   import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
   import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
+  import ImportRestDatasourceModal from "./ImportRestDatasourceModal.svelte"
 
   export let modal
   let integrations = []
   let integration = {}
   let internalTableModal
   let externalDatasourceModal
+  let importModal
+
+  $: showImportButton = false
+
+  checkShowImport()
 
   const INTERNAL = "BUDIBASE"
 
@@ -33,6 +39,15 @@
       config,
       schema: selected.datasource,
     }
+    checkShowImport()
+  }
+
+  function checkShowImport() {
+    showImportButton = integration.type === "REST"
+  }
+
+  function showImportModal() {
+    importModal.show()
   }
 
   function chooseNextModal() {
@@ -63,11 +78,20 @@
   <DatasourceConfigModal {integration} {modal} />
 </Modal>
 
+<Modal bind:this={importModal}>
+  {#if integration.type === "REST"}
+    <ImportRestDatasourceModal {integration} {modal} />
+  {/if}
+</Modal>
+
 <Modal bind:this={modal}>
   <ModalContent
     disabled={!Object.keys(integration).length}
     title="Data"
     confirmText="Continue"
+    showSecondaryButton={showImportButton}
+    secondaryButtonText="Import"
+    secondaryAction={() => showImportModal()}
     showCancelButton={false}
     size="M"
     onConfirm={() => {

packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestDatasourceModal.svelte

@@ -0,0 +1,103 @@
+<script>
+  import { goto } from "@roxi/routify"
+  import {
+    ModalContent,
+    notifications,
+    Body,
+    Layout,
+    Tabs,
+    Tab,
+    Input,
+    Heading,
+    TextArea,
+    Dropzone,
+  } from "@budibase/bbui"
+  import analytics, { Events } from "analytics"
+  import { datasources, queries } from "stores/backend"
+
+  export let modal
+
+  let data = {
+    url: "",
+    raw: "",
+    file: "",
+  }
+
+  let lastTouched = "url"
+
+  $: {
+    console.log({ data })
+    console.log({ lastTouched })
+  }
+
+  const getPayload = () => {
+    return {
+      type: lastTouched,
+      data: data[lastTouched],
+    }
+  }
+
+  async function importDatasource() {
+    try {
+      // Create datasource
+      const resp = await datasources.import(getPayload())
+
+      // // update the tables incase data source plus
+      await queries.fetch()
+      await datasources.select(resp._id)
+      $goto(`./datasource/${resp._id}`)
+      notifications.success(`Datasource imported successfully.`)
+      analytics.captureEvent(Events.DATASOURCE.IMPORTED, {
+        name: resp.name,
+        source: resp.source,
+      })
+      return true
+    } catch (err) {
+      notifications.error(`Error importing datasource: ${err}`)
+      return false
+    }
+  }
+</script>
+
+<ModalContent
+  onConfirm={() => importDatasource()}
+  onCancel={() => modal.show()}
+  confirmText={"Import"}
+  cancelText="Back"
+  size="L"
+>
+  <Layout noPadding>
+    <Heading size="S">Import</Heading>
+    <Body size="XS"
+      >Import your rest collection using one of the options below</Body
+    >
+    <Tabs selected="Link">
+      <Tab title="Link">
+        <Input
+          bind:value={data.url}
+          on:change={() => (lastTouched = "url")}
+          label="Enter a URL"
+          placeholder="e.g. https://petstore.swagger.io/v2/swagger.json"
+        />
+      </Tab>
+      <Tab title="File">
+        <Dropzone
+          bind:value={data.file}
+          on:change={() => (lastTouched = "file")}
+          fileTags={["OpenAPI", "Swagger 2.0"]}
+        />
+      </Tab>
+      <Tab title="Raw Text">
+        <TextArea
+          bind:value={data.raw}
+          on:change={() => (lastTouched = "raw")}
+          label={"Paste raw text"}
+          placeholder={'e.g. curl --location --request GET "https://example.com"'}
+        />
+      </Tab>
+    </Tabs>
+  </Layout>
+</ModalContent>
+
+<style>
+</style>

packages/builder/src/stores/backend/datasources.js

@@ -84,6 +84,11 @@ export function createDatasourcesStore() {
 
       return updateDatasource(response)
     },
+    import: async body => {
+      let response
+      response = await api.post(`/api/queries/import/swagger2`, body)
+      return updateDatasource(response)
+    },
     delete: async datasource => {
       const response = await api.delete(
         `/api/datasources/${datasource._id}/${datasource._rev}`

packages/builder/yarn.lock

@@ -969,10 +969,10 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/bbui@^0.9.185-alpha.12", "@budibase/bbui@^0.9.188":
+"@budibase/bbui@^0.9.185-alpha.21", "@budibase/bbui@^0.9.189":
-  version "0.9.188"
+  version "0.9.189"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.188.tgz#82c108172fbf81a84378e0ef4ca7cba61ea8d0ba"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.189.tgz#d5802e9b6aabccdef4205f0edfa7ed5616ac1aff"
-  integrity sha512-KevJxHdASITX9RzLvm+b2K3VMwqYFTumvrlpStAP6UIoyPkls0xaAc2KiJJ7Kkq48UkkBtAbOYaMxsFbAaTsbQ==
+  integrity sha512-YqM21mtrg8yTN9mqG4CnFfvoOelmhy3V69LyoITdQT6aGiwt/efHzknSlaUH3/0yLH9MuzwkHDzUmbe7QrsqEA==
   dependencies:
     "@adobe/spectrum-css-workflow-icons" "^1.2.1"
     "@spectrum-css/actionbutton" "^1.0.1"
@@ -1018,14 +1018,14 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/client@^0.9.185-alpha.12":
+"@budibase/client@^0.9.185-alpha.21":
-  version "0.9.188"
+  version "0.9.189"
-  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.9.188.tgz#c5bf6f3bccdb370b236b9e69e0118334ad3eccfd"
+  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.9.189.tgz#8f96b607f36bbb7390fd53b04360851f0c12aaac"
-  integrity sha512-yP2WLWb2yQAwPBxVzpNGjSHpATMZMzcxl2gK6vw662F7YC8xGFHNfZZqEwPvrVwnx+d7LFZR/kJxJOvvk7YCVw==
+  integrity sha512-L2i3CaQt4aFL7JKkRrEWWx8NemHTEOKLXvXq7LGM4u3GlcFIIkcL113EkXQT1bIZcf6AuuC2CfNsmZKioOPh2A==
   dependencies:
-    "@budibase/bbui" "^0.9.188"
+    "@budibase/bbui" "^0.9.189"
     "@budibase/standard-components" "^0.9.139"
-    "@budibase/string-templates" "^0.9.188"
+    "@budibase/string-templates" "^0.9.189"
     regexparam "^1.3.0"
     shortid "^2.2.15"
     svelte-spa-router "^3.0.5"
@@ -1080,10 +1080,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/string-templates@^0.9.185-alpha.12", "@budibase/string-templates@^0.9.188":
+"@budibase/string-templates@^0.9.185-alpha.21", "@budibase/string-templates@^0.9.189":
-  version "0.9.188"
+  version "0.9.189"
-  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.9.188.tgz#f37836ed23dbd2217cb157030ada7cd59f6a2165"
+  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.9.189.tgz#515a4ce85da550ce19d78c4c592ab5839a8ebe3a"
-  integrity sha512-O/bL0I5OJO9W2OizIe9vBHowCLwwASPBrsGiAIB8L0x6AivYMq8j1mvNRwLXZjpHTjv86bU/LyG/3CP837oDsg==
+  integrity sha512-JmnpuPx1CItNIFCMUxBz+4DVpYu96QxteU2Vi17pjWb0B7qsWwHkmcMmYbd+iTW4oxgOufbP8slfyfbu2XhbzA==
   dependencies:
     "@budibase/handlebars-helpers" "^0.11.7"
     dayjs "^1.10.4"

packages/server/src/api/controllers/query.js

@@ -5,6 +5,10 @@ const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
 const { Thread, ThreadType } = require("../../threads")
 
+const fetch = require("node-fetch")
+const Joi = require("joi")
+const { save: saveDatasource } = require("./datasource")
+
 const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
 
 // simple function to append "readable" to all read queries
@@ -30,6 +34,132 @@ exports.fetch = async function (ctx) {
   ctx.body = enrichQueries(body.rows.map(row => row.doc))
 }
 
+// const query = {
+//   datasourceId: "datasource_b9a474302a174d1295e4c273cd72bde9",
+//   name: "available pets (import)",
+//   parameters: [],
+//   fields: {
+//     headers: {},
+//     queryString: "status=available",
+//     path: "v2/pet/findByStatus",
+//   },
+//   queryVerb: "read",
+//   transformer: "return data",
+//   schema: {},
+//   readable: true
+// }
+
+function generateQueryValidation() {
+  // prettier-ignore
+  return Joi.object({
+    _id: Joi.string(),
+    _rev: Joi.string(),
+    name: Joi.string().required(),
+    fields: Joi.object().required(),
+    datasourceId: Joi.string().required(),
+    readable: Joi.boolean(),
+    parameters: Joi.array().items(Joi.object({
+      name: Joi.string(),
+      default: Joi.string().allow(""),
+    })),
+    queryVerb: Joi.string().allow().required(),
+    extra: Joi.object().optional(),
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
+  })
+}
+
+const verbs = {
+  get: "read",
+  post: "create",
+  put: "update",
+  patch: "patch",
+  delete: "delete",
+}
+
+const constructQuery = (datasource, swagger, path, method, config) => {
+  const query = {
+    datasourceId: datasource._id,
+  }
+  query.name = config.operationId || path
+  query.parameters = []
+  query.fields = {
+    headers: {},
+    // queryString: "status=available",
+    path: path,
+  }
+  query.transformer = "return data"
+  query.schema = {}
+  query.readable = true
+  query.queryVerb = verbs[method]
+
+  return query
+}
+
+// {
+//   "type": "url",
+//   "data": "www.url.com/swagger.json"
+// }
+
+exports.import = async function (ctx) {
+  const importConfig = ctx.request.body
+
+  let data
+
+  if (importConfig.type === "url") {
+    data = await fetch(importConfig.data).then(res => res.json())
+  } else if (importConfig.type === "raw") {
+    data = JSON.parse(importConfig.data)
+  }
+
+  const db = new CouchDB(ctx.appId)
+  const schema = generateQueryValidation()
+
+  // create datasource
+  const scheme = data.schemes.includes("https") ? "https" : "http"
+  const url = `${scheme}://${data.host}${data.basePath}`
+  const name = data.info.title
+
+  // TODO: Refactor datasource creation into shared function
+  const datasourceCtx = {
+    ...ctx,
+  }
+  datasourceCtx.request.body.datasource = {
+    type: "datasource",
+    source: "REST",
+    config: {
+      url: url,
+      defaultHeaders: {},
+    },
+    name: name,
+  }
+  await saveDatasource(datasourceCtx)
+  const datasource = datasourceCtx.body.datasource
+
+  // create query
+
+  for (const [path, method] of Object.entries(data.paths)) {
+    for (const [methodName, config] of Object.entries(method)) {
+      const query = constructQuery(datasource, data, path, methodName, config)
+
+      // validate query
+      const { error } = schema.validate(query)
+      if (error) {
+        ctx.throw(400, `Invalid - ${error.message}`)
+        return
+      }
+
+      // persist query
+      query._id = generateQueryID(query.datasourceId)
+      await db.put(query)
+    }
+  }
+
+  // return the datasource
+  ctx.body = { datasource }
+  ctx.status = 200
+}
+
 exports.save = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const query = ctx.request.body

packages/server/src/api/routes/query.js

@@ -57,6 +57,11 @@ router
     generateQueryValidation(),
     queryController.save
   )
+  .post(
+    "/api/queries/import/swagger2",
+    authorized(BUILDER),
+    queryController.import
+  )
   .post(
     "/api/queries/preview",
     bodyResource("datasourceId"),

packages/server/src/integrations/index.ts

@@ -40,9 +40,9 @@ const INTEGRATIONS = {
 }
 
 // optionally add oracle integration if the oracle binary can be installed
-if (!(process.arch === 'arm64' && process.platform === 'darwin')) {
+if (!(process.arch === "arm64" && process.platform === "darwin")) {
   const oracle = require("./oracle")
-  DEFINITIONS[SourceNames.ORACLE] =  oracle.schema
+  DEFINITIONS[SourceNames.ORACLE] = oracle.schema
   INTEGRATIONS[SourceNames.ORACLE] = oracle.integration
 }
 

packages/server/src/integrations/oracle.ts

@@ -352,14 +352,23 @@ module OracleModule {
      * Knex default returning behaviour does not work with oracle
      * Manually add the behaviour for the return column
      */
-    private addReturning(query: SqlQuery, bindings: BindParameters, returnColumn: string) {
+    private addReturning(
+      query: SqlQuery,
+      bindings: BindParameters,
+      returnColumn: string
+    ) {
       if (bindings instanceof Array) {
         bindings.push({ dir: oracledb.BIND_OUT })
-        query.sql = query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
+        query.sql =
+          query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
       }
     }
 
-    private async internalQuery<T>(query: SqlQuery, returnColum?: string, operation?: string): Promise<Result<T>> {
+    private async internalQuery<T>(
+      query: SqlQuery,
+      returnColum?: string,
+      operation?: string
+    ): Promise<Result<T>> {
       let connection
       try {
         connection = await this.getConnection()
@@ -367,7 +376,10 @@ module OracleModule {
         const options: ExecuteOptions = { autoCommit: true }
         const bindings: BindParameters = query.bindings || []
 
-        if (returnColum && (operation === Operation.CREATE || operation === Operation.UPDATE)) {
+        if (
+          returnColum &&
+          (operation === Operation.CREATE || operation === Operation.UPDATE)
+        ) {
           this.addReturning(query, bindings, returnColum)
         }
 
@@ -414,14 +426,14 @@ module OracleModule {
       return response.rows ? response.rows : []
     }
 
-    async update(query: SqlQuery | string): Promise<any[]>  {
+    async update(query: SqlQuery | string): Promise<any[]> {
       const response = await this.internalQuery(getSqlQuery(query))
       return response.rows && response.rows.length
         ? response.rows
         : [{ updated: true }]
     }
 
-    async delete(query: SqlQuery | string): Promise<any[]>  {
+    async delete(query: SqlQuery | string): Promise<any[]> {
       const response = await this.internalQuery(getSqlQuery(query))
       return response.rows && response.rows.length
         ? response.rows
@@ -431,8 +443,9 @@ module OracleModule {
     async query(json: QueryJson) {
       const primaryKeys = json.meta!.table!.primary
       const primaryKey = primaryKeys ? primaryKeys[0] : undefined
-      const queryFn = (query: any, operation: string) => this.internalQuery(query, primaryKey, operation)
+      const queryFn = (query: any, operation: string) =>
-      const processFn = (response: any) => response.rows ? response.rows : []
+        this.internalQuery(query, primaryKey, operation)
+      const processFn = (response: any) => (response.rows ? response.rows : [])
       const output = await this.queryWithReturning(json, queryFn, processFn)
       return output
     }

packages/server/src/integrations/utils.ts

@@ -2,7 +2,11 @@ import { SqlQuery } from "../definitions/datasource"
 import { Datasource, Table } from "../definitions/common"
 import { SourceNames } from "../definitions/datasource"
 const { DocumentTypes, SEPARATOR } = require("../db/utils")
-const { FieldTypes, BuildSchemaErrors, InvalidColumns } = require("../constants")
+const {
+  FieldTypes,
+  BuildSchemaErrors,
+  InvalidColumns,
+} = require("../constants")
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
@@ -42,7 +46,7 @@ export enum SqlClients {
   MS_SQL = "mssql",
   POSTGRES = "pg",
   MY_SQL = "mysql",
-  ORACLE = "oracledb"
+  ORACLE = "oracledb",
 }
 
 export function isExternalTable(tableId: string) {