Merge pull request #3970 from Budibase/fix/jan-various-fixes

Various fixes
5 years ago · bcdefafe44
8 changed files with 137 additions and 22 deletions
--- a/packages/auth/src/objectStore/index.js
+++ b/packages/auth/src/objectStore/index.js
@ -206,6 +206,34 @@ exports.retrieveToTmp = async (bucketName, filepath) => {
  return outputPath
 }

+/**
+ * Delete a single file.
+ */
+exports.deleteFile = async (bucketName, filepath) => {
+  const objectStore = exports.ObjectStore(bucketName)
+  await exports.makeSureBucketExists(objectStore, bucketName)
+  const params = {
+    Bucket: bucketName,
+    Key: filepath,
+  }
+  return objectStore.deleteObject(params)
+}
+
+exports.deleteFiles = async (bucketName, filepaths) => {
+  const objectStore = exports.ObjectStore(bucketName)
+  await exports.makeSureBucketExists(objectStore, bucketName)
+  const params = {
+    Bucket: bucketName,
+    Delete: {
+      Objects: filepaths.map(path => ({ Key: path })),
+    },
+  }
+  return objectStore.deleteObjects(params).promise()
+}
+
+/**
+ * Delete a path, including everything within.
+ */
 exports.deleteFolder = async (bucketName, folder) => {
  bucketName = sanitizeBucket(bucketName)
  folder = sanitizeKey(folder)
--- a/packages/builder/src/builderStore/datasource.js
+++ b/packages/builder/src/builderStore/datasource.js
@ -23,10 +23,10 @@ function prepareData(config) {
  return datasource
 }

-export async function saveDatasource(config) {
+export async function saveDatasource(config, skipFetch = false) {
  const datasource = prepareData(config)
  // Create datasource
-  const resp = await datasources.save(datasource, datasource.plus)
+  const resp = await datasources.save(datasource, !skipFetch && datasource.plus)

  // update the tables incase data source plus
  await tables.fetch()
--- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte
@ -199,18 +199,18 @@
  <Body>
    Tell budibase how your tables are related to get even more smart features.
  </Body>
-{/if}
-{#if relationshipInfo && relationshipInfo.length > 0}
-  <Table
-    on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
-    schema={relationshipSchema}
-    data={relationshipInfo}
-    allowEditColumns={false}
-    allowEditRows={false}
-    allowSelectRows={false}
-  />
-{:else}
-  <Body size="S"><i>No relationships configured.</i></Body>
+  {#if relationshipInfo && relationshipInfo.length > 0}
+    <Table
+      on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
+      schema={relationshipSchema}
+      data={relationshipInfo}
+      allowEditColumns={false}
+      allowEditRows={false}
+      allowSelectRows={false}
+    />
+  {:else}
+    <Body size="S"><i>No relationships configured.</i></Body>
+  {/if}
 {/if}

 <style>
--- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte
@ -5,22 +5,28 @@
  import { IntegrationNames } from "constants/backend"
  import cloneDeep from "lodash/cloneDeepWith"
  import { saveDatasource as save } from "builderStore/datasource"
+  import { onMount } from "svelte"

  export let integration
  export let modal

  // kill the reference so the input isn't saved
  let datasource = cloneDeep(integration)
+  let skipFetch = false

  async function saveDatasource() {
    try {
-      const resp = await save(datasource)
+      const resp = await save(datasource, skipFetch)
      $goto(`./datasource/${resp._id}`)
      notifications.success(`Datasource updated successfully.`)
    } catch (err) {
      notifications.error(`Error saving datasource: ${err}`)
    }
  }
+
+  onMount(() => {
+    skipFetch = false
+  })
 </script>

 <ModalContent
@ -28,9 +34,16 @@
  onConfirm={() => saveDatasource()}
  onCancel={() => modal.show()}
  confirmText={datasource.plus
-    ? "Fetch tables from database"
+    ? "Save and fetch tables"
    : "Save and continue to query"}
  cancelText="Back"
+  showSecondaryButton={datasource.plus}
+  secondaryButtonText={datasource.plus ? "Skip table fetch" : undefined}
+  secondaryAction={() => {
+    skipFetch = true
+    saveDatasource()
+    return true
+  }}
  size="L"
 >
  <Layout noPadding>
--- a/packages/server/src/api/controllers/row/internal.js
+++ b/packages/server/src/api/controllers/row/internal.js
@ -11,6 +11,7 @@ const {
  inputProcessing,
  outputProcessing,
  processAutoColumn,
+  cleanupAttachments,
 } = require("../../../utilities/rowProcessor")
 const { FieldTypes } = require("../../../constants")
 const { isEqual } = require("lodash")
@ -25,6 +26,7 @@ const {
  getFromDesignDoc,
  getFromMemoryDoc,
 } = require("../view/utils")
+const { cloneDeep } = require("lodash/fp")

 const CALCULATION_TYPES = {
  SUM: "sum",
@ -109,14 +111,14 @@ exports.patch = async ctx => {
  const inputs = ctx.request.body
  const tableId = inputs.tableId
  const isUserTable = tableId === InternalTables.USER_METADATA
-  let dbRow
+  let oldRow
  try {
-    dbRow = await db.get(inputs._id)
+    oldRow = await db.get(inputs._id)
  } catch (err) {
    if (isUserTable) {
      // don't include the rev, it'll be the global rev
      // this time
-      dbRow = {
+      oldRow = {
        _id: inputs._id,
      }
    } else {
@ -125,13 +127,14 @@ exports.patch = async ctx => {
  }
  let dbTable = await db.get(tableId)
  // need to build up full patch fields before coerce
+  let combinedRow = cloneDeep(oldRow)
  for (let key of Object.keys(inputs)) {
    if (!dbTable.schema[key]) continue
-    dbRow[key] = inputs[key]
+    combinedRow[key] = inputs[key]
  }

  // this returns the table and row incase they have been updated
-  let { table, row } = inputProcessing(ctx.user, dbTable, dbRow)
+  let { table, row } = inputProcessing(ctx.user, dbTable, combinedRow)
  const validateResult = await validate({
    row,
    table,
@ -149,6 +152,8 @@ exports.patch = async ctx => {
    tableId: row.tableId,
    table,
  })
+  // check if any attachments removed
+  await cleanupAttachments(appId, table, { oldRow, row })

  if (isUserTable) {
    // the row has been updated, need to put it into the ctx
@ -295,6 +300,8 @@ exports.destroy = async function (ctx) {
    row,
    tableId: row.tableId,
  })
+  // remove any attachments that were on the row from object storage
+  await cleanupAttachments(appId, table, { row })

  let response
  if (ctx.params.tableId === InternalTables.USER_METADATA) {
@ -341,6 +348,8 @@ exports.bulkDestroy = async ctx => {
  } else {
    await db.bulkDocs(rows.map(row => ({ ...row, _deleted: true })))
  }
+  // remove any attachments that were on the rows from object storage
+  await cleanupAttachments(appId, table, { rows })
  await Promise.all(updates)
  return { response: { ok: true }, rows }
 }
--- a/packages/server/src/integrations/base/sqlTable.ts
+++ b/packages/server/src/integrations/base/sqlTable.ts
@ -36,7 +36,7 @@ function generateSchema(
      case FieldTypes.STRING:
      case FieldTypes.OPTIONS:
      case FieldTypes.LONGFORM:
-        schema.string(key)
+        schema.text(key)
        break
      case FieldTypes.NUMBER:
        // if meta is specified then this is a junction table entry
--- a/packages/server/src/utilities/fileSystem/utilities.js
+++ b/packages/server/src/utilities/fileSystem/utilities.js
@ -2,6 +2,7 @@ const {
  ObjectStore,
  makeSureBucketExists,
  upload,
+  deleteFiles,
  streamUpload,
  retrieve,
  retrieveToTmp,
@ -28,3 +29,4 @@ exports.retrieveToTmp = retrieveToTmp
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball
+exports.deleteFiles = deleteFiles
--- a/packages/server/src/utilities/rowProcessor/index.js
+++ b/packages/server/src/utilities/rowProcessor/index.js
@ -3,6 +3,10 @@ const { cloneDeep } = require("lodash/fp")
 const { FieldTypes, AutoFieldSubTypes } = require("../../constants")
 const { attachmentsRelativeURL } = require("../index")
 const { processFormulas } = require("./utils")
+const { deleteFiles } = require("../../utilities/fileSystem/utilities")
+const { ObjectStoreBuckets } = require("../../constants")
+const { isProdAppID, getDeployedAppID, dbExists } = require("@budibase/auth/db")
+const CouchDB = require("../../db")

 const BASE_AUTO_ID = 1

@ -95,6 +99,23 @@ const TYPE_TRANSFORM_MAP = {
  },
 }

+/**
+ * Given the old state of the row and the new one after an update, this will
+ * find the keys that have been removed in the updated row.
+ */
+function getRemovedAttachmentKeys(oldRow, row, attachmentKey) {
+  if (!oldRow[attachmentKey]) {
+    return []
+  }
+  const oldKeys = oldRow[attachmentKey].map(attachment => attachment.key)
+  // no attachments in new row, all removed
+  if (!row[attachmentKey]) {
+    return oldKeys
+  }
+  const newKeys = row[attachmentKey].map(attachment => attachment.key)
+  return oldKeys.filter(key => newKeys.indexOf(key) === -1)
+}
+
 /**
 * This will update any auto columns that are found on the row/table with the correct information based on
 * time now and the current logged in user making the request.
@ -272,3 +293,45 @@ exports.outputProcessing = async (
  }
  return wasArray ? enriched : enriched[0]
 }
+
+/**
+ * Clean up any attachments that were attached to a row.
+ * @param {string} appId The ID of the app from which a row is being deleted.
+ * @param {object} table The table from which a row is being removed.
+ * @param {any} row optional - the row being removed.
+ * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
+ * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @return {Promise<void>} When all attachments have been removed this will return.
+ */
+exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
+  if (!isProdAppID(appId)) {
+    const prodAppId = getDeployedAppID(appId)
+    // if prod exists, then don't allow deleting
+    const exists = await dbExists(CouchDB, prodAppId)
+    if (exists) {
+      return
+    }
+  }
+  let files = []
+  function addFiles(row, key) {
+    if (row[key]) {
+      files = files.concat(row[key].map(attachment => attachment.key))
+    }
+  }
+  for (let [key, schema] of Object.entries(table.schema)) {
+    if (schema.type !== FieldTypes.ATTACHMENT) {
+      continue
+    }
+    // if updating, need to manage the differences
+    if (oldRow && row) {
+      files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
+    } else if (row) {
+      addFiles(row, key)
+    } else if (rows) {
+      rows.forEach(row => addFiles(row, key))
+    }
+  }
+  if (files.length > 0) {
+    return deleteFiles(ObjectStoreBuckets.APPS, files)
+  }
+}