Some refactoring to make it easier to find specific edge cases of sql clients.

packages/server/src/integrations/base/sql.ts

@@ -7,7 +7,7 @@ import {
   SearchFilters,
   SortDirection,
 } from "../../definitions/datasource"
-import { isIsoDateString } from "../utils"
+import { isIsoDateString, SqlClients } from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 
 const BASE_LIMIT = 5000
@@ -65,7 +65,7 @@ class InternalBuilder {
       iterate(filters.string, (key, value) => {
         const fnc = allOr ? "orWhere" : "where"
         // postgres supports ilike, nothing else does
-        if (this.client === "pg") {
+        if (this.client === SqlClients.POSTGRES) {
           query = query[fnc](key, "ilike", `${value}%`)
         } else {
           const rawFnc = `${fnc}Raw`
@@ -78,7 +78,7 @@ class InternalBuilder {
       iterate(filters.fuzzy, (key, value) => {
         const fnc = allOr ? "orWhere" : "where"
         // postgres supports ilike, nothing else does
-        if (this.client === "pg") {
+        if (this.client === SqlClients.POSTGRES) {
           query = query[fnc](key, "ilike", `%${value}%`)
         } else {
           const rawFnc = `${fnc}Raw`
@@ -216,7 +216,7 @@ class InternalBuilder {
         query = query.orderBy(key, direction)
       }
     }
-    if (this.client === "mssql" && !sort && paginate?.limit) {
+    if (this.client === SqlClients.MS_SQL && !sort && paginate?.limit) {
       // @ts-ignore
       query = query.orderBy(json.meta?.table?.primary[0])
     }
@@ -370,9 +370,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     // same as delete, manage returning
     if (operation === Operation.CREATE || operation === Operation.UPDATE) {
       let id
-      if (sqlClient === "mssql") {
+      if (sqlClient === SqlClients.MS_SQL) {
         id = results?.[0].id
-      } else if (sqlClient === "mysql") {
+      } else if (sqlClient === SqlClients.MY_SQL) {
         id = results?.insertId
       }
       row = processFn(

packages/server/src/integrations/microsoftSqlServer.ts

@@ -6,19 +6,19 @@ import {
   QueryTypes,
   SqlQuery,
 } from "../definitions/datasource"
-import { getSqlQuery } from "./utils"
+import {
+  getSqlQuery,
+  buildExternalTableId,
+  convertSqlType,
+  finaliseExternalTables,
+  SqlClients,
+} from "./utils"
 import { DatasourcePlus } from "./base/datasourcePlus"
 import { Table, TableSchema } from "../definitions/common"
 
 module MSSQLModule {
   const sqlServer = require("mssql")
   const Sql = require("./base/sql")
-  const { FieldTypes } = require("../constants")
-  const {
-    buildExternalTableId,
-    convertType,
-    finaliseExternalTables,
-  } = require("./utils")
 
   interface MSSQLConfig {
     user: string
@@ -79,31 +79,6 @@ module MSSQLModule {
     },
   }
 
-  // TODO: need to update this
-  const TYPE_MAP = {
-    text: FieldTypes.LONGFORM,
-    blob: FieldTypes.LONGFORM,
-    enum: FieldTypes.STRING,
-    varchar: FieldTypes.STRING,
-    float: FieldTypes.NUMBER,
-    int: FieldTypes.NUMBER,
-    numeric: FieldTypes.NUMBER,
-    bigint: FieldTypes.NUMBER,
-    mediumint: FieldTypes.NUMBER,
-    decimal: FieldTypes.NUMBER,
-    dec: FieldTypes.NUMBER,
-    double: FieldTypes.NUMBER,
-    real: FieldTypes.NUMBER,
-    fixed: FieldTypes.NUMBER,
-    smallint: FieldTypes.NUMBER,
-    timestamp: FieldTypes.DATETIME,
-    date: FieldTypes.DATETIME,
-    datetime: FieldTypes.DATETIME,
-    time: FieldTypes.DATETIME,
-    tinyint: FieldTypes.BOOLEAN,
-    json: DatasourceFieldTypes.JSON,
-  }
-
   async function internalQuery(
     client: any,
     query: SqlQuery,
@@ -175,7 +150,7 @@ module MSSQLModule {
     }
 
     constructor(config: MSSQLConfig) {
-      super("mssql")
+      super(SqlClients.MS_SQL)
       this.config = config
       const clientCfg = {
         ...this.config,
@@ -241,7 +216,7 @@ module MSSQLModule {
           if (typeof name !== "string") {
             continue
           }
-          const type: string = convertType(def.DATA_TYPE, TYPE_MAP)
+          const type: string = convertSqlType(def.DATA_TYPE)
 
           schema[name] = {
             autocolumn: !!autoColumns.find((col: string) => col === name),
@@ -256,7 +231,7 @@ module MSSQLModule {
           schema,
         }
       }
-      const final = finaliseExternalTables(tables)
+      const final = finaliseExternalTables(tables, entities)
       this.tables = final.tables
       this.schemaErrors = final.errors
     }

packages/server/src/integrations/mysql.ts

@@ -2,23 +2,22 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-  Operation,
   QueryJson,
   SqlQuery,
 } from "../definitions/datasource"
 import { Table, TableSchema } from "../definitions/common"
-import { getSqlQuery } from "./utils"
+import {
+  getSqlQuery,
+  SqlClients,
+  buildExternalTableId,
+  convertSqlType,
+  finaliseExternalTables,
+} from "./utils"
 import { DatasourcePlus } from "./base/datasourcePlus"
 
 module MySQLModule {
   const mysql = require("mysql2")
   const Sql = require("./base/sql")
-  const {
-    buildExternalTableId,
-    convertType,
-    finaliseExternalTables,
-  } = require("./utils")
-  const { FieldTypes } = require("../constants")
 
   interface MySQLConfig {
     host: string
@@ -29,30 +28,6 @@ module MySQLModule {
     ssl?: object
   }
 
-  const TYPE_MAP = {
-    text: FieldTypes.LONGFORM,
-    blob: FieldTypes.LONGFORM,
-    enum: FieldTypes.STRING,
-    varchar: FieldTypes.STRING,
-    float: FieldTypes.NUMBER,
-    int: FieldTypes.NUMBER,
-    numeric: FieldTypes.NUMBER,
-    bigint: FieldTypes.NUMBER,
-    mediumint: FieldTypes.NUMBER,
-    decimal: FieldTypes.NUMBER,
-    dec: FieldTypes.NUMBER,
-    double: FieldTypes.NUMBER,
-    real: FieldTypes.NUMBER,
-    fixed: FieldTypes.NUMBER,
-    smallint: FieldTypes.NUMBER,
-    timestamp: FieldTypes.DATETIME,
-    date: FieldTypes.DATETIME,
-    datetime: FieldTypes.DATETIME,
-    time: FieldTypes.DATETIME,
-    tinyint: FieldTypes.BOOLEAN,
-    json: DatasourceFieldTypes.JSON,
-  }
-
   const SCHEMA: Integration = {
     docs: "https://github.com/mysqljs/mysql",
     plus: true,
@@ -139,7 +114,7 @@ module MySQLModule {
     public schemaErrors: Record<string, string> = {}
 
     constructor(config: MySQLConfig) {
-      super("mysql")
+      super(SqlClients.MY_SQL)
       this.config = config
       if (config.ssl && Object.keys(config.ssl).length === 0) {
         delete config.ssl
@@ -184,7 +159,7 @@ module MySQLModule {
           schema[columnName] = {
             name: columnName,
             autocolumn: isAuto,
-            type: convertType(column.Type, TYPE_MAP),
+            type: convertSqlType(column.Type),
             constraints,
           }
         }

packages/server/src/integrations/postgres.ts

@@ -6,18 +6,18 @@ import {
   SqlQuery,
 } from "../definitions/datasource"
 import { Table } from "../definitions/common"
-import { getSqlQuery } from "./utils"
+import {
+  getSqlQuery,
+  buildExternalTableId,
+  convertSqlType,
+  finaliseExternalTables,
+  SqlClients,
+} from "./utils"
 import { DatasourcePlus } from "./base/datasourcePlus"
 
 module PostgresModule {
   const { Pool } = require("pg")
   const Sql = require("./base/sql")
-  const { FieldTypes } = require("../constants")
-  const {
-    buildExternalTableId,
-    convertType,
-    finaliseExternalTables,
-  } = require("./utils")
   const { escapeDangerousCharacters } = require("../utilities")
 
   const JSON_REGEX = /'{.*}'::json/s
@@ -97,22 +97,6 @@ module PostgresModule {
     },
   }
 
-  const TYPE_MAP = {
-    text: FieldTypes.LONGFORM,
-    varchar: FieldTypes.STRING,
-    integer: FieldTypes.NUMBER,
-    bigint: FieldTypes.NUMBER,
-    decimal: FieldTypes.NUMBER,
-    smallint: FieldTypes.NUMBER,
-    real: FieldTypes.NUMBER,
-    "double precision": FieldTypes.NUMBER,
-    timestamp: FieldTypes.DATETIME,
-    time: FieldTypes.DATETIME,
-    boolean: FieldTypes.BOOLEAN,
-    json: FieldTypes.JSON,
-    date: FieldTypes.DATETIME,
-  }
-
   async function internalQuery(client: any, query: SqlQuery) {
     // need to handle a specific issue with json data types in postgres,
     // new lines inside the JSON data will break it
@@ -154,7 +138,7 @@ module PostgresModule {
     `
 
     constructor(config: PostgresConfig) {
-      super("pg")
+      super(SqlClients.POSTGRES)
       this.config = config
 
       let newConfig = {
@@ -216,7 +200,7 @@ module PostgresModule {
           }
         }
 
-        const type: string = convertType(column.data_type, TYPE_MAP)
+        const type: string = convertSqlType(column.data_type)
         const identity = !!(
           column.identity_generation ||
           column.identity_start ||

packages/server/src/integrations/utils.ts

@@ -1,4 +1,4 @@
-import { Operation, SqlQuery } from "../definitions/datasource"
+import { SqlQuery } from "../definitions/datasource"
 import { Datasource, Table } from "../definitions/common"
 import { SourceNames } from "../definitions/datasource"
 const { DocumentTypes, SEPARATOR } = require("../db/utils")
@@ -7,6 +7,39 @@ const { FieldTypes, BuildSchemaErrors } = require("../constants")
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 
+const SQL_TYPE_MAP = {
+  text: FieldTypes.LONGFORM,
+  varchar: FieldTypes.STRING,
+  integer: FieldTypes.NUMBER,
+  bigint: FieldTypes.NUMBER,
+  decimal: FieldTypes.NUMBER,
+  smallint: FieldTypes.NUMBER,
+  real: FieldTypes.NUMBER,
+  "double precision": FieldTypes.NUMBER,
+  timestamp: FieldTypes.DATETIME,
+  time: FieldTypes.DATETIME,
+  boolean: FieldTypes.BOOLEAN,
+  json: FieldTypes.JSON,
+  date: FieldTypes.DATETIME,
+  blob: FieldTypes.LONGFORM,
+  enum: FieldTypes.STRING,
+  float: FieldTypes.NUMBER,
+  int: FieldTypes.NUMBER,
+  numeric: FieldTypes.NUMBER,
+  mediumint: FieldTypes.NUMBER,
+  dec: FieldTypes.NUMBER,
+  double: FieldTypes.NUMBER,
+  fixed: FieldTypes.NUMBER,
+  datetime: FieldTypes.DATETIME,
+  tinyint: FieldTypes.BOOLEAN,
+}
+
+export enum SqlClients {
+  MS_SQL = "mssql",
+  POSTGRES = "pg",
+  MY_SQL = "mysql",
+}
+
 export function isExternalTable(tableId: string) {
   return tableId.includes(DocumentTypes.DATASOURCE)
 }
@@ -68,8 +101,8 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
   }
 }
 
-export function convertType(type: string, map: { [key: string]: any }) {
-  for (let [external, internal] of Object.entries(map)) {
+export function convertSqlType(type: string) {
+  for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
     if (type.toLowerCase().includes(external)) {
       return internal
     }