Merge pull request #3437 from Budibase/oracle-datatasource-plus

Oracle datatasource plus

packages/server/__mocks__/oracledb.ts

@@ -0,0 +1,31 @@
+module OracleDbMock {
+  // mock execute
+  const execute = jest.fn(() => ({
+    rows: [
+      {
+        a: "string",
+        b: 1,
+      },
+    ],
+  }))
+
+  const close = jest.fn()
+
+  // mock connection
+  function Connection() {}
+  Connection.prototype.execute = execute
+  Connection.prototype.close = close
+
+  // mock oracledb
+  const oracleDb: any = {}
+  oracleDb.getConnection = jest.fn(() => {
+    // @ts-ignore
+    return new Connection()
+  })
+
+  // expose mocks
+  oracleDb.executeMock = execute
+  oracleDb.closeMock = close
+
+  module.exports = oracleDb
+}

packages/server/src/integrations/oracle.ts

@@ -3,14 +3,28 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
   SqlQuery,
+  QueryJson,
 } from "../definitions/datasource"
-import { getSqlQuery } from "./utils"
-import oracledb, { ExecuteOptions, Result,  Connection, ConnectionAttributes  } from "oracledb"
+import {
+  finaliseExternalTables,
+  getSqlQuery,
+  buildExternalTableId,
+  convertType,
+} from "./utils"
+import oracledb, {
+  ExecuteOptions,
+  Result,
+  Connection,
+  ConnectionAttributes,
+  BindParameters,
+} from "oracledb"
 import Sql from "./base/sql"
+import { Table } from "../definitions/common"
+import { DatasourcePlus } from "./base/datasourcePlus"
+import { FieldTypes } from "../constants"
 
 module OracleModule {
-
-  oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT;
+  oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
 
   interface OracleConfig {
     host: string
@@ -22,8 +36,10 @@ module OracleModule {
 
   const SCHEMA: Integration = {
     docs: "https://github.com/oracle/node-oracledb",
+    plus: true,
     friendlyName: "Oracle",
-    description: "Oracle Database is an object-relational database management system developed by Oracle Corporation",
+    description:
+      "Oracle Database is an object-relational database management system developed by Oracle Corporation",
     datasource: {
       host: {
         type: DatasourceFieldTypes.STRING,
@@ -46,7 +62,7 @@ module OracleModule {
       password: {
         type: DatasourceFieldTypes.PASSWORD,
         required: true,
-      }
+      },
     },
     query: {
       create: {
@@ -63,64 +79,361 @@ module OracleModule {
       },
     },
   }
-  class OracleIntegration extends Sql {
 
+  const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
+
+  const TYPE_MAP = {
+    long: FieldTypes.LONGFORM,
+    number: FieldTypes.NUMBER,
+    binary_float: FieldTypes.NUMBER,
+    binary_double: FieldTypes.NUMBER,
+    timestamp: FieldTypes.DATETIME,
+    date: FieldTypes.DATETIME,
+  }
+
+  /**
+   * Raw query response
+   */
+  interface ColumnsResponse {
+    TABLE_NAME: string
+    COLUMN_NAME: string
+    DATA_TYPE: string
+    DATA_DEFAULT: string | null
+    COLUMN_ID: number
+    CONSTRAINT_NAME: string | null
+    CONSTRAINT_TYPE: string | null
+    R_CONSTRAINT_NAME: string | null
+    SEARCH_CONDITION: string | null
+  }
+
+  /**
+   * An oracle constraint
+   */
+  interface OracleConstraint {
+    name: string
+    type: string
+    relatedConstraintName: string | null
+    searchCondition: string | null
+  }
+
+  /**
+   * An oracle column and it's related constraints
+   */
+  interface OracleColumn {
+    name: string
+    type: string
+    default: string | null
+    id: number
+    constraints: { [key: string]: OracleConstraint }
+  }
+
+  /**
+   * An oracle table and it's related columns
+   */
+  interface OracleTable {
+    name: string
+    columns: { [key: string]: OracleColumn }
+  }
+
+  const OracleContraintTypes = {
+    PRIMARY: "P",
+    NOT_NULL_OR_CHECK: "C",
+    FOREIGN_KEY: "R",
+    UNIQUE: "U",
+  }
+
+  class OracleIntegration extends Sql implements DatasourcePlus {
     private readonly config: OracleConfig
 
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
+
+    private readonly COLUMNS_SQL = `
+      SELECT
+        tabs.table_name,
+        cols.column_name,
+        cols.data_type,
+        cols.data_default,
+        cols.column_id,
+        cons.constraint_name,
+        cons.constraint_type,
+        cons.r_constraint_name,
+        cons.search_condition
+      FROM
+        user_tables tabs
+      JOIN 
+        user_tab_columns cols
+        ON tabs.table_name = cols.table_name 
+      LEFT JOIN 
+        user_cons_columns col_cons
+        ON cols.column_name = col_cons.column_name
+        AND cols.table_name = col_cons.table_name
+      LEFT JOIN 
+        user_constraints cons
+        ON col_cons.constraint_name = cons.constraint_name
+        AND cons.table_name = cols.table_name
+      WHERE
+        (cons.status = 'ENABLED'
+          OR cons.status IS NULL)
+    `
     constructor(config: OracleConfig) {
-      super("oracle")
+      super("oracledb")
       this.config = config
     }
 
-    private query = async (query: SqlQuery): Promise<Result<any>> => { 
+    /**
+     * Map the flat tabular columns and constraints data into a nested object
+     */
+    private mapColumns(result: Result<ColumnsResponse>): {
+      [key: string]: OracleTable
+    } {
+      const oracleTables: { [key: string]: OracleTable } = {}
+
+      if (result.rows) {
+        result.rows.forEach(row => {
+          const tableName = row.TABLE_NAME
+          const columnName = row.COLUMN_NAME
+          const dataType = row.DATA_TYPE
+          const dataDefault = row.DATA_DEFAULT
+          const columnId = row.COLUMN_ID
+          const constraintName = row.CONSTRAINT_NAME
+          const constraintType = row.CONSTRAINT_TYPE
+          const relatedConstraintName = row.R_CONSTRAINT_NAME
+          const searchCondition = row.SEARCH_CONDITION
+
+          let table = oracleTables[tableName]
+          if (!table) {
+            table = {
+              name: tableName,
+              columns: {},
+            }
+            oracleTables[tableName] = table
+          }
+
+          let column = table.columns[columnName]
+          if (!column) {
+            column = {
+              name: columnName,
+              type: dataType,
+              default: dataDefault,
+              id: columnId,
+              constraints: {},
+            }
+            table.columns[columnName] = column
+          }
+
+          if (constraintName && constraintType) {
+            let constraint = column.constraints[constraintName]
+            if (!constraint) {
+              constraint = {
+                name: constraintName,
+                type: constraintType,
+                relatedConstraintName: relatedConstraintName,
+                searchCondition: searchCondition,
+              }
+            }
+            column.constraints[constraintName] = constraint
+          }
+        })
+      }
+
+      return oracleTables
+    }
+
+    private isSupportedColumn(column: OracleColumn) {
+      if (UNSUPPORTED_TYPES.includes(column.type)) {
+        return false
+      }
+
+      return true
+    }
+
+    private isAutoColumn(column: OracleColumn) {
+      if (column.default && column.default.toLowerCase().includes("nextval")) {
+        return true
+      }
+
+      return false
+    }
+
+    /**
+     * No native boolean in oracle. Best we can do is to check if a manual 1 or 0 number constraint has been set up
+     * This matches the default behaviour for generating DDL used in knex.
+     */
+    private isBooleanType(column: OracleColumn): boolean {
+      if (
+        column.type.toLowerCase() === "number" &&
+        Object.values(column.constraints).filter(c => {
+          if (
+            c.type === OracleContraintTypes.NOT_NULL_OR_CHECK &&
+            c.searchCondition
+          ) {
+            const condition = c.searchCondition
+              .replace(/\s/g, "") // remove spaces
+              .replace(/[']+/g, "") // remove quotes
+            if (
+              condition.includes("in(0,1)") ||
+              condition.includes("in(1,0)")
+            ) {
+              return true
+            }
+          }
+          return false
+        }).length > 0
+      ) {
+        return true
+      }
+
+      return false
+    }
+
+    private internalConvertType(column: OracleColumn): string {
+      if (this.isBooleanType(column)) {
+        return FieldTypes.BOOLEAN
+      }
+
+      return convertType(column.type, TYPE_MAP)
+    }
+
+    /**
+     * Fetches the tables from the oracle table and assigns them to the datasource.
+     * @param {*} datasourceId - datasourceId to fetch
+     * @param entities - the tables that are to be built
+     */
+    async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+      const columnsResponse = await this.internalQuery<ColumnsResponse>({
+        sql: this.COLUMNS_SQL,
+      })
+      const oracleTables = this.mapColumns(columnsResponse)
+
+      const tables: { [key: string]: Table } = {}
+
+      // iterate each table
+      Object.values(oracleTables).forEach(oracleTable => {
+        let table = tables[oracleTable.name]
+        if (!table) {
+          table = {
+            _id: buildExternalTableId(datasourceId, oracleTable.name),
+            primary: [],
+            name: oracleTable.name,
+            schema: {},
+          }
+          tables[oracleTable.name] = table
+        }
+
+        // iterate each column on the table
+        Object.values(oracleTable.columns)
+          // remove columns that we can't read / save
+          .filter(oracleColumn => this.isSupportedColumn(oracleColumn))
+          // match the order of the columns in the db
+          .sort((c1, c2) => c1.id - c2.id)
+          .forEach(oracleColumn => {
+            const columnName = oracleColumn.name
+            let fieldSchema = table.schema[columnName]
+            if (!fieldSchema) {
+              fieldSchema = {
+                autocolumn: this.isAutoColumn(oracleColumn),
+                name: columnName,
+                type: this.internalConvertType(oracleColumn),
+              }
+              table.schema[columnName] = fieldSchema
+            }
+
+            // iterate each constraint on the column
+            Object.values(oracleColumn.constraints).forEach(
+              oracleConstraint => {
+                if (oracleConstraint.type === OracleContraintTypes.PRIMARY) {
+                  table.primary!.push(columnName)
+                }
+              }
+            )
+          })
+      })
+
+      const final = finaliseExternalTables(tables, entities)
+      this.tables = final.tables
+      this.schemaErrors = final.errors
+    }
+
+    private async internalQuery<T>(query: SqlQuery): Promise<Result<T>> {
       let connection
       try {
         connection = await this.getConnection()
 
-        const options : ExecuteOptions = { autoCommit: true }
-        const result: Result<any> = await connection.execute(query.sql, [], options)
+        const options: ExecuteOptions = { autoCommit: true }
+        const bindings: BindParameters = query.bindings || []
+        const result: Result<T> = await connection.execute<T>(
+          query.sql,
+          bindings,
+          options
+        )
 
         return result
       } finally {
-       if (connection) {
-         try {
-           await connection.close();
-         } catch (err) {
-           console.error(err);
-         }
-       }
+        if (connection) {
+          try {
+            await connection.close()
+          } catch (err) {
+            console.error(err)
+          }
+        }
       }
     }
 
     private getConnection = async (): Promise<Connection> => {
       //connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))"
-      const connectString = `${this.config.host}:${this.config.port || 1521}/${this.config.database}`
+      const connectString = `${this.config.host}:${this.config.port || 1521}/${
+        this.config.database
+      }`
       const attributes: ConnectionAttributes = {
         user: this.config.user,
         password: this.config.user,
         connectString,
       }
-      return oracledb.getConnection(attributes);
+      return oracledb.getConnection(attributes)
     }
 
     async create(query: SqlQuery | string) {
-      const response = await this.query(getSqlQuery(query))
-      return response.rows && response.rows.length ? response.rows : [{ created: true }]
+      const response = await this.internalQuery(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ created: true }]
     }
 
     async read(query: SqlQuery | string) {
-      const response = await this.query(getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.rows
     }
 
     async update(query: SqlQuery | string) {
-      const response = await this.query(getSqlQuery(query))
-      return response.rows && response.rows.length ? response.rows : [{ updated: true }]
+      const response = await this.internalQuery(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ updated: true }]
     }
 
     async delete(query: SqlQuery | string) {
-      const response = await this.query(getSqlQuery(query))
-      return response.rows && response.rows.length ? response.rows : [{ deleted: true }]
+      const response = await this.internalQuery(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ deleted: true }]
+    }
+
+    async query(json: QueryJson) {
+      const operation = this._operation(json).toLowerCase()
+      const input = this._query(json, { disableReturning: true })
+      if (Array.isArray(input)) {
+        const responses = []
+        for (let query of input) {
+          responses.push(await this.internalQuery(query))
+        }
+        return responses
+      } else {
+        const response = await this.internalQuery(input)
+        return response.rows && response.rows.length
+          ? response.rows
+          : [{ [operation]: true }]
+      }
     }
   }
 

packages/server/src/integrations/tests/oracle.spec.js

@@ -0,0 +1,94 @@
+const oracledb = require("oracledb")
+const OracleIntegration = require("../oracle")
+jest.mock("oracledb")
+
+class TestConfiguration {
+  constructor(config = {}) {
+    this.integration = new OracleIntegration.integration(config) 
+  }
+}
+
+const options = { autoCommit: true }
+
+describe("Oracle Integration", () => {
+  let config 
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    expect(oracledb.closeMock).toHaveBeenCalled()
+    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the create method with the correct params", async () => {
+    const sql = "insert into users (name, age) values ('Joe', 123);"
+    await config.integration.create({
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the read method with the correct params", async () => {
+    const sql = "select * from users;"
+    await config.integration.read({
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the update method with the correct params", async () => {
+    const sql = "update table users set name = 'test';"
+    const response = await config.integration.update({ 
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the delete method with the correct params", async () => {
+    const sql = "delete from users where name = 'todelete';"
+    await config.integration.delete({
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  describe("no rows returned", () => {
+    beforeEach(() => {
+      oracledb.executeMock.mockImplementation(() => ({ rows: [] }))
+    })
+
+    it("returns the correct response when the create response has no rows", async () => {
+    const sql = "insert into users (name, age) values ('Joe', 123);"
+      const response = await config.integration.create({ 
+        sql
+      })
+      expect(response).toEqual([{ created: true }])
+      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+    })
+
+    it("returns the correct response when the update response has no rows", async () => {
+      const sql = "update table users set name = 'test';"
+      const response = await config.integration.update({ 
+        sql
+      })
+      expect(response).toEqual([{ updated: true }])
+      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+    })
+
+    it("returns the correct response when the delete response has no rows", async () => {
+      const sql = "delete from users where name = 'todelete';"
+      const response = await config.integration.delete({ 
+        sql
+      })
+      expect(response).toEqual([{ deleted: true }])
+      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+    })
+  })
+})

packages/server/src/integrations/utils.ts

@@ -161,7 +161,7 @@ export function finaliseExternalTables(
   tables: { [key: string]: any },
   entities: { [key: string]: any }
 ) {
-  const finalTables: { [key: string]: any } = {}
+  let finalTables: { [key: string]: any } = {}
   const errors: { [key: string]: string } = {}
   for (let [name, table] of Object.entries(tables)) {
     // make sure every table has a key
@@ -172,5 +172,9 @@ export function finaliseExternalTables(
     // make sure all previous props have been added back
     finalTables[name] = copyExistingPropsOver(name, table, entities)
   }
+  // sort the tables by name
+  finalTables = Object.entries(finalTables)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .reduce((r, [k, v]) => ({ ...r, [k]: v }), {})
   return { tables: finalTables, errors }
 }