Merge pull request #3356 from Budibase/feature/existing-table-import

Import CSV to existing table from builder
5 years ago · e257b5ee8b
22 changed files with 279 additions and 74 deletions
--- a/packages/bbui/src/InlineAlert/InlineAlert.svelte
+++ b/packages/bbui/src/InlineAlert/InlineAlert.svelte
@ -47,5 +47,6 @@
    --spectrum-semantic-positive-border-color: #2d9d78;
    --spectrum-semantic-positive-icon-color: #2d9d78;
    --spectrum-semantic-negative-icon-color: #e34850;
    min-width: 100px;
  }
 </style>
--- a/packages/builder/src/components/backend/DataTable/DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte
@ -6,6 +6,7 @@
  import CreateViewButton from "./buttons/CreateViewButton.svelte"
  import ExistingRelationshipButton from "./buttons/ExistingRelationshipButton.svelte"
  import ExportButton from "./buttons/ExportButton.svelte"
  import ImportButton from "./buttons/ImportButton.svelte"
  import EditRolesButton from "./buttons/EditRolesButton.svelte"
  import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
  import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
@ -124,6 +125,10 @@
      <HideAutocolumnButton bind:hideAutocolumns />
      <!-- always have the export last -->
      <ExportButton view={$tables.selected?._id} />
      <ImportButton
        tableId={$tables.selected?._id}
        on:updaterows={onUpdateRows}
      />
      {#key id}
        <TableFilterButton {schema} on:change={onFilter} />
      {/key}
--- a/packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte
@ -7,7 +7,7 @@
  let modal
 </script>
-<ActionButton icon="Download" size="S" quiet on:click={modal.show}>
+<ActionButton icon="DataDownload" size="S" quiet on:click={modal.show}>
  Export
 </ActionButton>
 <Modal bind:this={modal}>
--- a/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte
@ -0,0 +1,15 @@
 <script>
  import { ActionButton, Modal } from "@budibase/bbui"
  import ImportModal from "../modals/ImportModal.svelte"
  export let tableId
  let modal
 </script>
 <ActionButton icon="DataUpload" size="S" quiet on:click={modal.show}>
  Import
 </ActionButton>
 <Modal bind:this={modal}>
  <ImportModal {tableId} on:updaterows />
 </Modal>
--- a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte
@ -0,0 +1,43 @@
 <script>
  import { ModalContent, Label, notifications, Body } from "@budibase/bbui"
  import TableDataImport from "../../TableNavigator/TableDataImport.svelte"
  import api from "builderStore/api"
  import { createEventDispatcher } from "svelte"
  const dispatch = createEventDispatcher()
  export let tableId
  let dataImport
  $: valid = dataImport?.csvString != null && dataImport?.valid
  async function importData() {
    const response = await api.post(`/api/tables/${tableId}/import`, {
      dataImport,
    })
    if (response.status !== 200) {
      const error = await response.text()
      notifications.error(`Unable to import data - ${error}`)
    } else {
      notifications.success("Rows successfully imported.")
    }
    dispatch("updaterows")
  }
 </script>
 <ModalContent
  title="Import Data"
  confirmText="Import"
  onConfirm={importData}
  disabled={!valid}
 >
  <Body
    >Import rows to an existing table from a CSV. Only columns from the CSV
    which exist in the table will be imported.</Body
  >
  <Label grey extraSmall>CSV to import</Label>
  <TableDataImport bind:dataImport bind:existingTableId={tableId} />
 </ModalContent>
 <style>
 </style>
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@ -1,6 +1,5 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Select, InlineAlert, notifications } from "@budibase/bbui"
  import { notifications } from "@budibase/bbui"
  import { FIELDS } from "constants/backend"
  import api from "builderStore/api"
@ -12,11 +11,13 @@
    valid: true,
    schema: {},
  }
  export let existingTableId
-  let csvString
+  let csvString = undefined
-  let primaryDisplay
+  let primaryDisplay = undefined
  let schema = {}
  let fields = []
  let hasValidated = false
  $: valid = !schema || fields.every(column => schema[column].success)
  $: dataImport = {
@ -25,6 +26,9 @@
    csvString,
    primaryDisplay,
  }
  $: noFieldsError = existingTableId
    ? "No columns in CSV match existing table schema"
    : "Could not find any columns to import"
  function buildTableSchema(schema) {
    const tableSchema = {}
@ -46,6 +50,7 @@
    const response = await api.post("/api/tables/csv/validate", {
      csvString,
      schema: schema || {},
      tableId: existingTableId,
    })
    const parseResult = await response.json()
@ -63,6 +68,7 @@
      notifications.error("CSV Invalid, please try another CSV file")
      return []
    }
    hasValidated = true
  }
  async function handleFile(evt) {
@ -138,6 +144,7 @@
          placeholder={null}
          getOptionLabel={option => option.label}
          getOptionValue={option => option.value}
          disabled={!!existingTableId}
        />
        <span class="field-status" class:error={!schema[columnName].success}>
          {schema[columnName].success ? "Success" : "Failure"}
@ -149,15 +156,22 @@
      </div>
    {/each}
  </div>
-{/if}
+  {#if !existingTableId}
-
+    <div class="display-column">
-{#if fields.length}
+      <Select
-  <div class="display-column">
+        label="Display Column"
-    <Select
+        bind:value={primaryDisplay}
-      label="Display Column"
+        options={fields}
-      bind:value={primaryDisplay}
+        sort
-      options={fields}
+      />
-      sort
+    </div>
  {/if}
 {:else if hasValidated}
  <div>
    <InlineAlert
      header="Invalid CSV"
      bind:message={noFieldsError}
      type="error"
    />
  </div>
 {/if}
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@ -33,6 +33,7 @@ interface RunConfig {
  sort?: SortJson
  paginate?: PaginationJson
  row?: Row
  rows?: Row[]
 }
 module External {
@ -600,7 +601,10 @@ module External {
        throw `Unable to process query, table "${tableName}" not defined.`
      }
      // look for specific components of config which may not be considered acceptable
-      let { id, row, filters, sort, paginate } = cleanupConfig(config, table)
+      let { id, row, filters, sort, paginate, rows } = cleanupConfig(
        config,
        table
      )
      filters = buildFilters(id, filters || {}, table)
      const relationships = this.buildRelationships(table)
      // clean up row on ingress using schema
@ -626,7 +630,7 @@ module External {
        sort,
        paginate,
        relationships,
-        body: row,
+        body: row || rows,
        // pass an id filter into extra, purely for mysql/returning
        extra: {
          idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
--- a/packages/server/src/api/controllers/row/external.js
+++ b/packages/server/src/api/controllers/row/external.js
@ -30,6 +30,8 @@ async function handleRequest(appId, operation, tableId, opts = {}) {
  )
 }
 exports.handleRequest = handleRequest
 exports.patch = async ctx => {
  const appId = ctx.appId
  const inputs = ctx.request.body
--- a/packages/server/src/api/controllers/table/external.js
+++ b/packages/server/src/api/controllers/table/external.js
@ -17,6 +17,8 @@ const {
 } = require("../../../constants")
 const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { cloneDeep } = require("lodash/fp")
 const csvParser = require("../../../utilities/csvParser")
 const { handleRequest } = require("../row/external")
 async function makeTableRequest(
  datasource,
@ -279,3 +281,20 @@ exports.destroy = async function (ctx) {
  return tableToDelete
 }
 exports.bulkImport = async function (ctx) {
  const appId = ctx.appId
  const table = await getTable(appId, ctx.params.tableId)
  const { dataImport } = ctx.request.body
  if (!dataImport || !dataImport.schema || !dataImport.csvString) {
    ctx.throw(400, "Provided data import information is invalid.")
  }
  const rows = await csvParser.transform({
    ...dataImport,
    existingTable: table,
  })
  await handleRequest(appId, DataSourceOperation.BULK_CREATE, table._id, {
    rows,
  })
  return table
 }
--- a/packages/server/src/api/controllers/table/index.js
+++ b/packages/server/src/api/controllers/table/index.js
@ -81,8 +81,26 @@ exports.destroy = async function (ctx) {
  ctx.body = { message: `Table ${tableId} deleted.` }
 }
 exports.bulkImport = async function (ctx) {
  const tableId = ctx.params.tableId
  await pickApi({ tableId }).bulkImport(ctx)
  // right now we don't trigger anything for bulk import because it
  // can only be done in the builder, but in the future we may need to
  // think about events for bulk items
  ctx.status = 200
  ctx.body = { message: `Bulk rows created.` }
 }
 exports.validateCSVSchema = async function (ctx) {
-  const { csvString, schema = {} } = ctx.request.body
+  // tableId being specified means its an import to an existing table
-  const result = await csvParser.parse(csvString, schema)
+  const { csvString, schema = {}, tableId } = ctx.request.body
  let existingTable
  if (tableId) {
    existingTable = await getTable(ctx.appId, tableId)
  }
  let result = await csvParser.parse(csvString, schema)
  if (existingTable) {
    result = csvParser.updateSchema({ schema: result, existingTable })
  }
  ctx.body = { schema: result }
 }
--- a/packages/server/src/api/controllers/table/internal.js
+++ b/packages/server/src/api/controllers/table/internal.js
@ -2,7 +2,12 @@ const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const { getRowParams, generateTableID } = require("../../../db/utils")
 const { FieldTypes } = require("../../../constants")
-const { TableSaveFunctions, hasTypeChanged } = require("./utils")
+const {
  TableSaveFunctions,
  hasTypeChanged,
  getTable,
  handleDataImport,
 } = require("./utils")
 exports.save = async function (ctx) {
  const appId = ctx.appId
@ -140,3 +145,11 @@ exports.destroy = async function (ctx) {
  return tableToDelete
 }
 exports.bulkImport = async function (ctx) {
  const appId = ctx.appId
  const table = await getTable(appId, ctx.params.tableId)
  const { dataImport } = ctx.request.body
  await handleDataImport(appId, ctx.user, table, dataImport)
  return table
 }
--- a/packages/server/src/api/controllers/table/utils.js
+++ b/packages/server/src/api/controllers/table/utils.js
@ -72,43 +72,47 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
 }
 exports.handleDataImport = async (appId, user, table, dataImport) => {
  if (!dataImport || !dataImport.csvString) {
    return table
  }
  const db = new CouchDB(appId)
-  if (dataImport && dataImport.csvString) {
+  // Populate the table with rows imported from CSV in a bulk update
-    // Populate the table with rows imported from CSV in a bulk update
+  const data = await csvParser.transform({
-    const data = await csvParser.transform(dataImport)
+    ...dataImport,
-
+    existingTable: table,
-    let finalData = []
+  })
-    for (let i = 0; i < data.length; i++) {
+
-      let row = data[i]
+  let finalData = []
-      row._id = generateRowID(table._id)
+  for (let i = 0; i < data.length; i++) {
-      row.tableId = table._id
+    let row = data[i]
-      const processed = inputProcessing(user, table, row, {
+    row._id = generateRowID(table._id)
-        noAutoRelationships: true,
+    row.tableId = table._id
-      })
+    const processed = inputProcessing(user, table, row, {
-      table = processed.table
+      noAutoRelationships: true,
-      row = processed.row
+    })
-
+    table = processed.table
-      for (let [fieldName, schema] of Object.entries(table.schema)) {
+    row = processed.row
-        // check whether the options need to be updated for inclusion as part of the data import
+
-        if (
+    for (let [fieldName, schema] of Object.entries(table.schema)) {
-          schema.type === FieldTypes.OPTIONS &&
+      // check whether the options need to be updated for inclusion as part of the data import
-          (!schema.constraints.inclusion ||
+      if (
-            schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
+        schema.type === FieldTypes.OPTIONS &&
-        ) {
+        (!schema.constraints.inclusion ||
-          schema.constraints.inclusion = [
+          schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
-            ...schema.constraints.inclusion,
+      ) {
-            row[fieldName],
+        schema.constraints.inclusion = [
-          ]
+          ...schema.constraints.inclusion,
-        }
+          row[fieldName],
        ]
      }
      finalData.push(row)
    }
-    await db.bulkDocs(finalData)
+    finalData.push(row)
    let response = await db.put(table)
    table._rev = response._rev
  }
  await db.bulkDocs(finalData)
  let response = await db.put(table)
  table._rev = response._rev
  return table
 }
--- a/packages/server/src/api/routes/table.js
+++ b/packages/server/src/api/routes/table.js
@ -53,5 +53,16 @@ router
    authorized(BUILDER),
    tableController.destroy
  )
  // this is currently builder only, but in the future
  // it could be carried out by an end user in app,
  // however some thought will need to be had about
  // implications for automations (triggers)
  // new trigger type, bulk rows created
  .post(
    "/api/tables/:tableId/import",
    paramResource("tableId"),
    authorized(BUILDER),
    tableController.bulkImport
  )
 module.exports = router
--- a/packages/server/src/api/routes/tests/misc.spec.js
+++ b/packages/server/src/api/routes/tests/misc.spec.js
@ -75,7 +75,11 @@ describe("run misc tests", () => {
        },
      })
      const dataImport = {
-        csvString: "a,b,c,d\n1,2,3,4"
+        csvString: "a,b,c,d\n1,2,3,4",
        schema: {},
      }
      for (let col of ["a", "b", "c", "d"]) {
        dataImport.schema[col] = { type: "string" }
      }
      await tableUtils.handleDataImport(
        config.getAppId(),
--- a/packages/server/src/constants/index.js
+++ b/packages/server/src/constants/index.js
@ -69,6 +69,7 @@ exports.DataSourceOperation = {
  READ: "READ",
  UPDATE: "UPDATE",
  DELETE: "DELETE",
  BULK_CREATE: "BULK_CREATE",
  CREATE_TABLE: "CREATE_TABLE",
  UPDATE_TABLE: "UPDATE_TABLE",
  DELETE_TABLE: "DELETE_TABLE",
--- a/packages/server/src/definitions/datasource.ts
+++ b/packages/server/src/definitions/datasource.ts
@ -1,10 +1,11 @@
-import { Table } from "./common"
+import { Row, Table } from "./common"
 export enum Operation {
  CREATE = "CREATE",
  READ = "READ",
  UPDATE = "UPDATE",
  DELETE = "DELETE",
  BULK_CREATE = "BULK_CREATE",
  CREATE_TABLE = "CREATE_TABLE",
  UPDATE_TABLE = "UPDATE_TABLE",
  DELETE_TABLE = "DELETE_TABLE",
@ -144,7 +145,7 @@ export interface QueryJson {
  filters?: SearchFilters
  sort?: SortJson
  paginate?: PaginationJson
-  body?: object
+  body?: Row | Row[]
  table?: Table
  meta?: {
    table?: Table
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@ -179,6 +179,16 @@ class InternalBuilder {
    }
  }
  bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
    const { endpoint, body } = json
    let query: KnexQuery = knex(endpoint.entityId)
    if (!Array.isArray(body)) {
      return query
    }
    const parsedBody = body.map(row => parseBody(row))
    return query.insert(parsedBody)
  }
  read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
    let { endpoint, resource, filters, sort, paginate, relationships } = json
    const tableName = endpoint.entityId
@ -294,6 +304,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
      case Operation.DELETE:
        query = builder.delete(client, json, opts)
        break
      case Operation.BULK_CREATE:
        query = builder.bulkCreate(client, json)
        break
      case Operation.CREATE_TABLE:
      case Operation.UPDATE_TABLE:
      case Operation.DELETE_TABLE:
--- a/packages/server/src/integrations/base/sqlTable.ts
+++ b/packages/server/src/integrations/base/sqlTable.ts
@ -29,10 +29,7 @@ function generateSchema(
  for (let [key, column] of Object.entries(table.schema)) {
    // skip things that are already correct
    const oldColumn = oldTable ? oldTable.schema[key] : null
-    if (
+    if ((oldColumn && oldColumn.type) || (primaryKey === key && !isJunction)) {
      (oldColumn && oldColumn.type) ||
      (primaryKey === key && !isJunction)
    ) {
      continue
    }
    switch (column.type) {
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@ -130,7 +130,7 @@ module PostgresModule {
    public tables: Record<string, Table> = {}
    public schemaErrors: Record<string, string> = {}
-    COLUMNS_SQL!: string 
+    COLUMNS_SQL!: string
    PRIMARY_KEYS_SQL = `
    select tc.table_schema, tc.table_name, kc.column_name as primary_key 
@ -165,11 +165,11 @@ module PostgresModule {
    setSchema() {
      if (!this.config.schema) {
-        this.config.schema = 'public'
+        this.config.schema = "public"
      }
-      this.client.on('connect', (client: any) => {    
+      this.client.on("connect", (client: any) => {
-        client.query(`SET search_path TO ${this.config.schema}`);
+        client.query(`SET search_path TO ${this.config.schema}`)
-      });
+      })
      this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
    }
--- a/packages/server/src/utilities/csvParser.js
+++ b/packages/server/src/utilities/csvParser.js
@ -51,7 +51,7 @@ function parse(csvString, parsers) {
    })
    result.subscribe(row => {
      // For each CSV row parse all the columns that need parsed
-      for (let key in parsers) {
+      for (let key of Object.keys(parsers)) {
        if (!schema[key] || schema[key].success) {
          // get the validator for the column type
          const validator = VALIDATORS[parsers[key].type]
@ -76,16 +76,58 @@ function parse(csvString, parsers) {
  })
 }
-async function transform({ schema, csvString }) {
+function updateSchema({ schema, existingTable }) {
  if (!schema) {
    return schema
  }
  const finalSchema = {}
  const schemaKeyMap = {}
  Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
  for (let [key, field] of Object.entries(existingTable.schema)) {
    const lcKey = key.toLowerCase()
    const foundKey = schemaKeyMap[lcKey]
    if (foundKey) {
      finalSchema[key] = schema[foundKey]
      finalSchema[key].type = field.type
    }
  }
  return finalSchema
 }
 async function transform({ schema, csvString, existingTable }) {
  const colParser = {}
-  for (let key in schema) {
+  // make sure the table has all the columns required for import
  if (existingTable) {
    schema = updateSchema({ schema, existingTable })
  }
  for (let key of Object.keys(schema)) {
    colParser[key] = PARSERS[schema[key].type] || schema[key].type
  }
  try {
-    const json = await csv({ colParser }).fromString(csvString)
+    const data = await csv({ colParser }).fromString(csvString)
-    return json
+    const schemaKeyMap = {}
    Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
    for (let element of data) {
      if (!data) {
        continue
      }
      for (let key of Object.keys(element)) {
        const mappedKey = schemaKeyMap[key.toLowerCase()]
        // isn't a column in the table, remove it
        if (mappedKey == null) {
          delete element[key]
        }
        // casing is different, fix it in row
        else if (key !== mappedKey) {
          element[mappedKey] = element[key]
          delete element[key]
        }
      }
    }
    return data
  } catch (err) {
    console.error(`Error transforming CSV to JSON for data import`, err)
    throw err
@ -95,4 +137,5 @@ async function transform({ schema, csvString }) {
 module.exports = {
  parse,
  transform,
  updateSchema,
 }
--- a/packages/server/src/utilities/tests/snapshots/csvParser.spec.js.snap
+++ b/packages/server/src/utilities/tests/snapshots/csvParser.spec.js.snap
@ -3,19 +3,13 @@
 exports[`CSV Parser transformation transforms a CSV file into JSON 1`] = `
 Array [
  Object {
    "Address": "5 Sesame Street",
    "Age": 4324,
    "Name": "Bertå",
  },
  Object {
    "Address": "1 World Trade Center",
    "Age": 34,
    "Name": "Ernie",
  },
  Object {
    "Address": "44 Second Avenue",
    "Age": 23423,
    "Name": "Big Bird",
  },
 ]
 `;
--- a/packages/server/src/utilities/tests/csvParser.spec.js
+++ b/packages/server/src/utilities/tests/csvParser.spec.js
@ -24,6 +24,9 @@ const SCHEMAS = {
    Age: {
      type: "omit",
    },
    Name: {
      type: "string",
    },
  },
  BROKEN: {
    Address: {