Merge pull request #3541 from Budibase/fix/sql-issues

Fixing various SQL issues

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -38,6 +38,7 @@
   const LINK_TYPE = FIELDS.LINK.type
   const STRING_TYPE = FIELDS.STRING.type
   const NUMBER_TYPE = FIELDS.NUMBER.type
+  const DATE_TYPE = FIELDS.DATETIME.type
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@@ -256,6 +257,9 @@
     ) {
       fieldToCheck.constraints.numericality = {}
     }
+    if (fieldToCheck.type === DATE_TYPE && !fieldToCheck.constraints.datetime) {
+      fieldToCheck.constraints.datetime = {}
+    }
   }
 
   function checkErrors(fieldInfo) {

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte

@@ -1,6 +1,6 @@
 <script>
   import { goto } from "@roxi/routify"
-  import { datasources } from "stores/backend"
+  import { datasources, tables } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import { ActionMenu, MenuItem, Icon } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
@@ -13,10 +13,16 @@
 
   async function deleteDatasource() {
     const wasSelectedSource = $datasources.selected
+    const wasSelectedTable = $tables.selected
     await datasources.delete(datasource)
     notifications.success("Datasource deleted")
     // navigate to first index page if the source you are deleting is selected
-    if (wasSelectedSource === datasource._id) {
+    const entities = Object.values(datasource.entities)
+    if (
+      wasSelectedSource === datasource._id ||
+      (entities &&
+        entities.find(entity => entity._id === wasSelectedTable?._id))
+    ) {
       $goto("./datasource")
     }
   }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -197,9 +197,9 @@ module External {
     return row
   }
 
-  function isMany(field: FieldSchema) {
+  function isOneSide(field: FieldSchema) {
     return (
-      field.relationshipType && field.relationshipType.split("-")[0] === "many"
+      field.relationshipType && field.relationshipType.split("-")[0] === "one"
     )
   }
 
@@ -274,25 +274,37 @@ module External {
         const linkTable = this.tables[linkTableName]
         // @ts-ignore
         const linkTablePrimary = linkTable.primary[0]
-        if (!isMany(field)) {
+        // one to many
+        if (isOneSide(field)) {
           newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(
             row[key][0]
           )[0]
-        } else {
+        }
+        // many to many
+        else if (field.through) {
           // we're not inserting a doc, will be a bunch of update calls
-          const isUpdate = !field.through
-          const thisKey: string = isUpdate
-            ? "id"
-            : field.throughTo || linkTablePrimary
-          // @ts-ignore
-          const otherKey: string = isUpdate
-            ? field.fieldName
-            : field.throughFrom || tablePrimary
+          const otherKey: string = field.throughFrom || linkTablePrimary
+          const thisKey: string = field.throughTo || tablePrimary
           row[key].map((relationship: any) => {
-            // we don't really support composite keys for relationships, this is why [0] is used
             manyRelationships.push({
               tableId: field.through || field.tableId,
-              isUpdate,
+              isUpdate: false,
+              key: otherKey,
+              [otherKey]: breakRowIdField(relationship)[0],
+              // leave the ID for enrichment later
+              [thisKey]: `{{ literal ${tablePrimary} }}`,
+            })
+          })
+        }
+        // many to one
+        else {
+          const thisKey: string = "id"
+          // @ts-ignore
+          const otherKey: string = field.fieldName
+          row[key].map((relationship: any) => {
+            manyRelationships.push({
+              tableId: field.tableId,
+              isUpdate: true,
               key: otherKey,
               [thisKey]: breakRowIdField(relationship)[0],
               // leave the ID for enrichment later
@@ -425,8 +437,8 @@ module External {
           )
           definition.through = throughTableName
           // don't support composite keys for relationships
-          definition.from = field.throughFrom || table.primary[0]
-          definition.to = field.throughTo || linkTable.primary[0]
+          definition.from = field.throughTo || table.primary[0]
+          definition.to = field.throughFrom || linkTable.primary[0]
           definition.fromPrimary = table.primary[0]
           definition.toPrimary = linkTable.primary[0]
         }
@@ -448,24 +460,36 @@ module External {
       // make a new request to get the row with all its relationships
       // we need this to work out if any relationships need removed
       for (let field of Object.values(table.schema)) {
-        if (field.type !== FieldTypes.LINK || !field.fieldName) {
+        if (
+          field.type !== FieldTypes.LINK ||
+          !field.fieldName ||
+          isOneSide(field)
+        ) {
           continue
         }
         const isMany = field.relationshipType === RelationshipTypes.MANY_TO_MANY
         const tableId = isMany ? field.through : field.tableId
-        const manyKey = field.throughFrom || primaryKey
+        const { tableName: relatedTableName } = breakExternalTableId(tableId)
+        // @ts-ignore
+        const linkPrimaryKey = this.tables[relatedTableName].primary[0]
+        const manyKey = field.throughTo || primaryKey
+        const lookupField = isMany ? primaryKey : field.foreignKey
         const fieldName = isMany ? manyKey : field.fieldName
+        if (!lookupField || !row[lookupField]) {
+          continue
+        }
         const response = await getDatasourceAndQuery(this.appId, {
           endpoint: getEndpoint(tableId, DataSourceOperation.READ),
           filters: {
             equal: {
-              [fieldName]: row[primaryKey],
+              [fieldName]: row[lookupField],
             },
           },
         })
         // this is the response from knex if no rows found
         const rows = !response[0].read ? response : []
-        related[fieldName] = { rows, isMany, tableId }
+        const storeTo = isMany ? field.throughFrom || linkPrimaryKey : manyKey
+        related[storeTo] = { rows, isMany, tableId }
       }
       return related
     }

packages/server/src/integrations/base/sql.ts

@@ -21,6 +21,9 @@ function parse(input: any) {
   if (Array.isArray(input)) {
     return JSON.stringify(input)
   }
+  if (input == undefined) {
+    return null
+  }
   if (typeof input !== "string") {
     return input
   }
@@ -43,7 +46,10 @@ function parseBody(body: any) {
   return body
 }
 
-function parseFilters(filters: SearchFilters): SearchFilters {
+function parseFilters(filters: SearchFilters | undefined): SearchFilters {
+  if (!filters) {
+    return {}
+  }
   for (let [key, value] of Object.entries(filters)) {
     let parsed
     if (typeof value === "object") {
@@ -152,6 +158,23 @@ class InternalBuilder {
     return query
   }
 
+  addSorting(query: KnexQuery, json: QueryJson): KnexQuery {
+    let { sort, paginate } = json
+    if (!sort) {
+      return query
+    }
+    const table = json.meta?.table
+    for (let [key, value] of Object.entries(sort)) {
+      const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
+      query = query.orderBy(`${table?.name}.${key}`, direction)
+    }
+    if (this.client === SqlClients.MS_SQL && !sort && paginate?.limit) {
+      // @ts-ignore
+      query = query.orderBy(`${table?.name}.${table?.primary[0]}`)
+    }
+    return query
+  }
+
   addRelationships(
     knex: Knex,
     query: KnexQuery,
@@ -249,22 +272,18 @@ class InternalBuilder {
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    if (sort) {
-      for (let [key, value] of Object.entries(sort)) {
-        const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
-        query = query.orderBy(key, direction)
-      }
-    }
-    if (this.client === SqlClients.MS_SQL && !sort && paginate?.limit) {
-      // @ts-ignore
-      query = query.orderBy(json.meta?.table?.primary[0])
-    }
     query = this.addFilters(tableName, query, filters)
+    // add sorting to pre-query
+    query = this.addSorting(query, json)
     // @ts-ignore
     let preQuery: KnexQuery = knex({
       // @ts-ignore
       [tableName]: query,
     }).select(selectStatement)
+    // have to add after as well (this breaks MS-SQL)
+    if (this.client !== SqlClients.MS_SQL) {
+      preQuery = this.addSorting(preQuery, json)
+    }
     // handle joins
     return this.addRelationships(
       knex,

packages/server/src/integrations/index.ts

@@ -40,9 +40,9 @@ const INTEGRATIONS = {
 }
 
 // optionally add oracle integration if the oracle binary can be installed
-if (!(process.arch === 'arm64' && process.platform === 'darwin')) {
+if (!(process.arch === "arm64" && process.platform === "darwin")) {
   const oracle = require("./oracle")
-  DEFINITIONS[SourceNames.ORACLE] =  oracle.schema
+  DEFINITIONS[SourceNames.ORACLE] = oracle.schema
   INTEGRATIONS[SourceNames.ORACLE] = oracle.integration
 }
 

packages/server/src/integrations/mysql.ts

@@ -134,7 +134,9 @@ module MySQLModule {
         false
       )
       const tableNames = tablesResp.map(
-        (obj: any) => obj[`Tables_in_${database.toLowerCase()}`]
+        (obj: any) =>
+          obj[`Tables_in_${database}`] ||
+          obj[`Tables_in_${database.toLowerCase()}`]
       )
       for (let tableName of tableNames) {
         const primaryKeys = []

packages/server/src/integrations/oracle.ts

@@ -352,14 +352,23 @@ module OracleModule {
      * Knex default returning behaviour does not work with oracle
      * Manually add the behaviour for the return column
      */
-    private addReturning(query: SqlQuery, bindings: BindParameters, returnColumn: string) {
+    private addReturning(
+      query: SqlQuery,
+      bindings: BindParameters,
+      returnColumn: string
+    ) {
       if (bindings instanceof Array) {
         bindings.push({ dir: oracledb.BIND_OUT })
-        query.sql = query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
+        query.sql =
+          query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
       }
     }
 
-    private async internalQuery<T>(query: SqlQuery, returnColum?: string, operation?: string): Promise<Result<T>> {
+    private async internalQuery<T>(
+      query: SqlQuery,
+      returnColum?: string,
+      operation?: string
+    ): Promise<Result<T>> {
       let connection
       try {
         connection = await this.getConnection()
@@ -367,7 +376,10 @@ module OracleModule {
         const options: ExecuteOptions = { autoCommit: true }
         const bindings: BindParameters = query.bindings || []
 
-        if (returnColum && (operation === Operation.CREATE || operation === Operation.UPDATE)) {
+        if (
+          returnColum &&
+          (operation === Operation.CREATE || operation === Operation.UPDATE)
+        ) {
           this.addReturning(query, bindings, returnColum)
         }
 
@@ -414,14 +426,14 @@ module OracleModule {
       return response.rows ? response.rows : []
     }
 
-    async update(query: SqlQuery | string): Promise<any[]>  {
+    async update(query: SqlQuery | string): Promise<any[]> {
       const response = await this.internalQuery(getSqlQuery(query))
       return response.rows && response.rows.length
         ? response.rows
         : [{ updated: true }]
     }
 
-    async delete(query: SqlQuery | string): Promise<any[]>  {
+    async delete(query: SqlQuery | string): Promise<any[]> {
       const response = await this.internalQuery(getSqlQuery(query))
       return response.rows && response.rows.length
         ? response.rows
@@ -431,8 +443,9 @@ module OracleModule {
     async query(json: QueryJson) {
       const primaryKeys = json.meta!.table!.primary
       const primaryKey = primaryKeys ? primaryKeys[0] : undefined
-      const queryFn = (query: any, operation: string) => this.internalQuery(query, primaryKey, operation)
-      const processFn = (response: any) => response.rows ? response.rows : []
+      const queryFn = (query: any, operation: string) =>
+        this.internalQuery(query, primaryKey, operation)
+      const processFn = (response: any) => (response.rows ? response.rows : [])
       const output = await this.queryWithReturning(json, queryFn, processFn)
       return output
     }

packages/server/src/integrations/utils.ts

@@ -2,7 +2,11 @@ import { SqlQuery } from "../definitions/datasource"
 import { Datasource, Table } from "../definitions/common"
 import { SourceNames } from "../definitions/datasource"
 const { DocumentTypes, SEPARATOR } = require("../db/utils")
-const { FieldTypes, BuildSchemaErrors, InvalidColumns } = require("../constants")
+const {
+  FieldTypes,
+  BuildSchemaErrors,
+  InvalidColumns,
+} = require("../constants")
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
@@ -42,7 +46,7 @@ export enum SqlClients {
   MS_SQL = "mssql",
   POSTGRES = "pg",
   MY_SQL = "mysql",
-  ORACLE = "oracledb"
+  ORACLE = "oracledb",
 }
 
 export function isExternalTable(tableId: string) {