budibase postgres E2E working

packages/builder/src/components/backend/DataTable/ExternalDataSourceTable.svelte

@@ -0,0 +1,32 @@
+<script>
+  import { backendUiStore } from "builderStore"
+  import * as api from "./api"
+  import Table from "./Table.svelte"
+  import EditIntegrationConfigButton from "./buttons/EditIntegrationConfigButton.svelte"
+
+  let data = []
+  let loading = false
+
+  $: table = $backendUiStore.selectedTable
+  $: title = table.name
+  $: schema = table.schema
+  $: tableView = {
+    schema,
+    name: $backendUiStore.selectedView.name,
+  }
+
+  // Fetch rows for specified table
+  $: {
+    if ($backendUiStore.selectedView?.name?.startsWith("all_")) {
+      loading = true
+      api.fetchDataForView($backendUiStore.selectedView).then(rows => {
+        data = rows || []
+        loading = false
+      })
+    }
+  }
+</script>
+
+<Table {title} {schema} {data} allowEditing={true} {loading}>
+  <EditIntegrationConfigButton {table} />
+</Table>

packages/builder/src/components/backend/DataTable/buttons/EditIntegrationConfigButton.svelte

@@ -0,0 +1,43 @@
+<script>
+  import {
+    DropdownMenu,
+    TextButton as Button,
+    Icon,
+    Modal,
+    ModalContent,
+  } from "@budibase/bbui"
+  import { backendUiStore } from "builderStore"
+  import api from "builderStore/api"
+  import EditIntegrationConfig from "../modals/EditIntegrationConfig.svelte"
+
+  export let table
+
+  $: console.log("The table config is", table)
+
+  let modal
+
+  // TODO: revisit
+  async function saveTable() {
+    const SAVE_TABLE_URL = `/api/tables`
+    const response = await api.post(SAVE_TABLE_URL, table)
+    const savedTable = await response.json()
+    await backendUiStore.actions.tables.fetch()
+    backendUiStore.actions.tables.select(savedTable)
+  }
+</script>
+
+<div>
+  <Button text small on:click={modal.show}>
+    <Icon name="edit" />
+    Configure Datasource
+  </Button>
+</div>
+<Modal bind:this={modal}>
+  <ModalContent
+    confirmText="Save"
+    cancelText="Cancel"
+    onConfirm={saveTable}
+    title={'Edit Datasource Configuration'}>
+    <EditIntegrationConfig onClosed={modal.hide} bind:table />
+  </ModalContent>
+</Modal>

packages/builder/src/components/backend/DataTable/modals/EditIntegrationConfig.svelte

@@ -0,0 +1,69 @@
+<script>
+  import { Select, Button, Input, TextArea, Heading } from "@budibase/bbui"
+  import { FIELDS } from "constants/backend"
+  import { backendUiStore } from "builderStore"
+
+  export let table
+
+  let fields = []
+
+  $: {
+    const schema = {}
+    for (let field of fields) {
+      if (!field.name) continue
+      schema[field.name] = FIELDS[field.type]
+    }
+    table.schema = schema
+  }
+
+  function newField() {
+    fields = [...fields, {}]
+  }
+
+  function deleteField(idx) {
+    fields.splice(idx, 1)
+    fields = fields
+  }
+</script>
+
+<form>
+  <Heading extraSmall black>Schema</Heading>
+  <!-- {#each Object.keys(table.schema) as schemaKey, idx}
+    <Input
+      thin
+      type={'text'}
+      label={schemaKey}
+      bind:value={table.schema[schemaKey]} />
+  {/each} -->
+  {#each fields as field}
+    <div class="field">
+      <Input thin type={'text'} bind:value={field.name} />
+      <Select secondary thin bind:value={field.type}>
+        <option value={''}>Select an option</option>
+        <option value={'STRING'}>Text</option>
+        <option value={'NUMBER'}>Number</option>
+        <option value={'BOOLEAN'}>Boolean</option>
+        <option value={'DATETIME'}>Datetime</option>
+      </Select>
+    </div>
+  {/each}
+
+  <Button thin secondary on:click={newField}>Add Field</Button>
+  <Heading extraSmall black>Datasource</Heading>
+  {#each Object.keys(table.integration) as configKey}
+    <Input
+      thin
+      type={configKey.type}
+      label={configKey}
+      bind:value={table.integration[configKey]} />
+  {/each}
+</form>
+
+<style>
+  .field {
+    display: grid;
+    grid-gap: 10px;
+    grid-template-columns: 1fr 1fr;
+    margin-bottom: var(--spacing-xs);
+  }
+</style>

packages/builder/src/components/backend/TableNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte

@@ -0,0 +1,15 @@
+<script>
+  import { Input, TextArea } from "@budibase/bbui"
+
+  export let integration
+</script>
+
+<form>
+  {#each Object.keys(integration) as configKey}
+    <Input
+      thin
+      type={configKey.type}
+      label={configKey}
+      bind:value={integration[configKey]} />
+  {/each}
+</form>

packages/builder/src/components/backend/TableNavigator/TableIntegrationMenu/index.svelte

@@ -0,0 +1,52 @@
+<script>
+  import { onMount } from "svelte"
+  import { Input, TextArea } from "@budibase/bbui"
+  import api from "builderStore/api"
+
+  export let integration = {}
+
+  let integrationsPromise = fetchIntegrations()
+  let selectedIntegration
+
+  async function fetchIntegrations() {
+    const INTEGRATIONS_URL = `/api/integrations`
+    const response = await api.get(INTEGRATIONS_URL)
+    const json = await response.json()
+    return json
+  }
+</script>
+
+<section>
+  {#await integrationsPromise}
+    Loading integrations...
+  {:then integrations}
+    {#each Object.keys(integrations) as integrationType}
+      <div
+        on:click={() => {
+          selectedIntegration = integrations[integrationType]
+          integration.type = integrationType
+        }}>
+        <h6>{integrationType}</h6>
+      </div>
+    {/each}
+  {:catch}
+    shit itself
+  {/await}
+
+  {#if selectedIntegration}
+    {#each Object.keys(selectedIntegration) as configKey}
+      <Input
+        thin
+        type={configKey.type}
+        label={configKey}
+        bind:value={integration[configKey]} />
+    {/each}
+    <TextArea label={'Query'} bind:value={integration.query} />
+  {/if}
+</section>
+
+<style>
+  section {
+    display: grid;
+  }
+</style>

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -4,6 +4,7 @@
   import { notifier } from "builderStore/store/notifications"
   import { Input, Label, ModalContent } from "@budibase/bbui"
   import TableDataImport from "../TableDataImport.svelte"
+  import TableIntegrationMenu from "../TableIntegrationMenu/index.svelte"
   import analytics from "analytics"
   import screenTemplates from "builderStore/store/screenTemplates"
   import { NEW_ROW_TEMPLATE } from "builderStore/store/screenTemplates/newRowScreen"
@@ -19,6 +20,7 @@
   let modal
   let name
   let dataImport
+  let integration
   let error = ""
 
   function checkValid(evt) {
@@ -35,6 +37,7 @@
       name,
       schema: dataImport.schema || {},
       dataImport,
+      integration
     }
 
     // Only set primary display if defined
@@ -88,4 +91,8 @@
     <Label grey extraSmall>Create Table from CSV (Optional)</Label>
     <TableDataImport bind:dataImport />
   </div>
+  <div>
+    <Label grey extraSmall>Create Integrated Table from External Source</Label>
+    <TableIntegrationMenu bind:integration />
+  </div>
 </ModalContent>

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -3,6 +3,7 @@
   import { notifier } from "builderStore/store/notifications"
   import { DropdownMenu, Button, Input } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import IntegrationConfigForm from "../TableIntegrationMenu//IntegrationConfigForm.svelte" 
   import { DropdownContainer, DropdownItem } from "components/common/Dropdowns"
 
   export let table
@@ -39,7 +40,7 @@
 
   async function deleteTable() {
     await backendUiStore.actions.tables.delete(table)
-    store.store.actions.screens.delete(templateScreens)
+    store.actions.screens.delete(templateScreens)
     await backendUiStore.actions.tables.fetch()
     notifier.success("Table deleted")
     hideEditor()
@@ -78,6 +79,9 @@
           bind:value={table.name}
           on:input={checkValid}
           {error} />
+        {#if table.integration}
+          <IntegrationConfigForm integration={table.integration} />
+        {/if}
         <footer>
           <Button secondary on:click={hideEditor}>Cancel</Button>
           <Button primary disabled={error} on:click={save}>Save</Button>

packages/builder/src/pages/[application]/data/table/[selectedTable]/index.svelte

@@ -1,13 +1,20 @@
 <script>
   import TableDataTable from "components/backend/DataTable/DataTable.svelte"
+  import ExternalDataSourceTable from "components/backend/DataTable/ExternalDataSourceTable.svelte"
   import { backendUiStore } from "builderStore"
 
   $: selectedTable = $backendUiStore.selectedTable
 </script>
 
 {#if $backendUiStore.selectedDatabase._id && selectedTable.name}
-  <TableDataTable />
+  {#if selectedTable.integration}
-{:else}<i>Create your first table to start building</i>{/if}
+    <ExternalDataSourceTable />
+  {:else}
+    <TableDataTable />
+  {/if}
+{:else}
+  <i>Create your first table to start building</i>
+{/if}
 
 <style>
   i {

packages/server/package.json

@@ -81,6 +81,7 @@
     "lodash": "^4.17.13",
     "mustache": "^4.0.1",
     "node-fetch": "^2.6.0",
+    "pg": "^8.5.1",
     "pino-pretty": "^4.0.0",
     "pouchdb": "^7.2.1",
     "pouchdb-all-dbs": "^1.0.2",

packages/server/src/api/controllers/integration.js

@@ -0,0 +1,8 @@
+const { definitions } = require("../../integrations")
+
+exports.fetch = async function(ctx) {
+  // TODO: fetch these from a github repo etc
+  console.log(definitions)
+  ctx.status = 200
+  ctx.body = definitions
+}

packages/server/src/api/controllers/row.js

@@ -8,6 +8,7 @@ const {
   SEPARATOR,
 } = require("../../db/utils")
 const { cloneDeep } = require("lodash")
+const { integrations } = require("../../integrations")
 
 const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}`
 
@@ -145,7 +146,7 @@ exports.fetchView = async function(ctx) {
   const viewName = ctx.params.viewName
 
   // if this is a table view being looked for just transfer to that
-  if (viewName.indexOf(TABLE_VIEW_BEGINS_WITH) === 0) {
+  if (viewName.startsWith(TABLE_VIEW_BEGINS_WITH)) {
     ctx.params.tableId = viewName.substring(4)
     await exports.fetchTableRows(ctx)
     return
@@ -186,6 +187,15 @@ exports.fetchView = async function(ctx) {
 exports.fetchTableRows = async function(ctx) {
   const appId = ctx.user.appId
   const db = new CouchDB(appId)
+
+  const table = await db.get(ctx.params.tableId)
+
+  if (table.integration) {
+    const Integration = integrations[table.integration.type]
+    ctx.body = await new Integration(table.integration).query()
+    return
+  }
+
   const response = await db.allDocs(
     getRowParams(ctx.params.tableId, null, {
       include_docs: true,

packages/server/src/api/routes/index.js

@@ -16,6 +16,7 @@ const apiKeysRoutes = require("./apikeys")
 const templatesRoutes = require("./templates")
 const analyticsRoutes = require("./analytics")
 const routingRoutes = require("./routing")
+const integrationRoutes = require("./integration")
 
 exports.mainRoutes = [
   deployRoutes,
@@ -32,6 +33,7 @@ exports.mainRoutes = [
   analyticsRoutes,
   webhookRoutes,
   routingRoutes,
+  integrationRoutes,
   // these need to be handled last as they still use /api/:tableId
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,

packages/server/src/api/routes/integration.js

@@ -0,0 +1,10 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/integration")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("../../utilities/security/permissions")
+
+const router = Router()
+
+router.get("/api/integrations", authorized(BUILDER), controller.fetch)
+
+module.exports = router

packages/server/src/integrations/Integration.js

@@ -0,0 +1,7 @@
+class Integration {
+  definition() {
+    return this.options
+  }
+}
+
+module.exports = Integration

packages/server/src/integrations/index.js

@@ -0,0 +1,14 @@
+const postgres = require("./postgres")
+
+const DEFINITIONS = {
+  POSTGRES: postgres.schema,
+}
+
+const INTEGRATIONS = {
+  POSTGRES: postgres.integration,
+}
+
+module.exports = {
+  definitions: DEFINITIONS,
+  integrations: INTEGRATIONS,
+}

packages/server/src/integrations/postgres.js

@@ -0,0 +1,48 @@
+const { Client } = require("pg")
+
+const POSTGRES_OPTIONS = {
+  host: {
+    type: "string",
+    required: true,
+    default: "localhost",
+  },
+  port: {
+    type: "number",
+    required: true,
+    default: 5432,
+  },
+  database: {
+    type: "string",
+    default: "postgres",
+  },
+  username: {
+    type: "string",
+    default: "root",
+  },
+  password: {
+    type: "password",
+    default: "root",
+  },
+}
+
+class PostgresIntegration {
+  constructor(config) {
+    this.config = config
+    this.client = new Client(config)
+    this.connect()
+  }
+
+  async connect() {
+    return this.client.connect()
+  }
+
+  async query() {
+    const response = await this.client.query(this.config.query)
+    return response.rows
+  }
+}
+
+module.exports = {
+  schema: POSTGRES_OPTIONS,
+  integration: PostgresIntegration,
+}

packages/server/yarn.lock

@@ -1588,6 +1588,11 @@ buffer-from@1.1.1, buffer-from@^1.0.0:
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
   integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
 
+buffer-writer@2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-2.0.0.tgz#ce7eb81a38f7829db09c873f2fbb792c0c98ec04"
+  integrity sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==
+
 buffer@4.9.2:
   version "4.9.2"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8"
@@ -5883,6 +5888,11 @@ package-json@^6.3.0:
     registry-url "^5.0.0"
     semver "^6.2.0"
 
+packet-reader@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74"
+  integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==
+
 pako@^1.0.5:
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
@@ -6000,6 +6010,57 @@ performance-now@^2.1.0:
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
   integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
 
+pg-connection-string@^2.4.0:
+  version "2.4.0"
+  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.4.0.tgz#c979922eb47832999a204da5dbe1ebf2341b6a10"
+  integrity sha512-3iBXuv7XKvxeMrIgym7njT+HlZkwZqqGX4Bu9cci8xHZNT+Um1gWKqCsAzcC0d95rcKMU5WBg6YRUcHyV0HZKQ==
+
+pg-int8@1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
+  integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==
+
+pg-pool@^3.2.2:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.2.2.tgz#a560e433443ed4ad946b84d774b3f22452694dff"
+  integrity sha512-ORJoFxAlmmros8igi608iVEbQNNZlp89diFVx6yV5v+ehmpMY9sK6QgpmgoXbmkNaBAx8cOOZh9g80kJv1ooyA==
+
+pg-protocol@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.4.0.tgz#43a71a92f6fe3ac559952555aa3335c8cb4908be"
+  integrity sha512-El+aXWcwG/8wuFICMQjM5ZSAm6OWiJicFdNYo+VY3QP+8vI4SvLIWVe51PppTzMhikUJR+PsyIFKqfdXPz/yxA==
+
+pg-types@^2.1.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3"
+  integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==
+  dependencies:
+    pg-int8 "1.0.1"
+    postgres-array "~2.0.0"
+    postgres-bytea "~1.0.0"
+    postgres-date "~1.0.4"
+    postgres-interval "^1.1.0"
+
+pg@^8.5.1:
+  version "8.5.1"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.5.1.tgz#34dcb15f6db4a29c702bf5031ef2e1e25a06a120"
+  integrity sha512-9wm3yX9lCfjvA98ybCyw2pADUivyNWT/yIP4ZcDVpMN0og70BUWYEGXPCTAQdGTAqnytfRADb7NERrY1qxhIqw==
+  dependencies:
+    buffer-writer "2.0.0"
+    packet-reader "1.0.0"
+    pg-connection-string "^2.4.0"
+    pg-pool "^3.2.2"
+    pg-protocol "^1.4.0"
+    pg-types "^2.1.0"
+    pgpass "1.x"
+
+pgpass@1.x:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.4.tgz#85eb93a83800b20f8057a2b029bf05abaf94ea9c"
+  integrity sha512-YmuA56alyBq7M59vxVBfPJrGSozru8QAdoNlWuW3cz8l+UX3cWge0vTvjKhsSHSJpo3Bom8/Mm6hf0TR5GY0+w==
+  dependencies:
+    split2 "^3.1.1"
+
 phin@^2.9.1:
   version "2.9.3"
   resolved "https://registry.yarnpkg.com/phin/-/phin-2.9.3.tgz#f9b6ac10a035636fb65dfc576aaaa17b8743125c"
@@ -6116,6 +6177,28 @@ posix-character-classes@^0.1.0:
   resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
   integrity sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=
 
+postgres-array@~2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e"
+  integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==
+
+postgres-bytea@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35"
+  integrity sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=
+
+postgres-date@~1.0.4:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8"
+  integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==
+
+postgres-interval@^1.1.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695"
+  integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==
+  dependencies:
+    xtend "^4.0.0"
+
 pouch-stream@^0.4.0:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/pouch-stream/-/pouch-stream-0.4.1.tgz#0c6d8475c9307677627991a2f079b301c3b89bdd"