mirror of https://github.com/Budibase/budibase.git
Rory Powell
5 years ago
13 changed files with 547 additions and 345 deletions
@ -1,334 +0,0 @@ |
|||
import CouchDB from "../../../db" |
|||
import { queryValidation } from "./validation" |
|||
import { generateQueryID } from "../../../db/utils" |
|||
import { Spec as Swagger2, Operation } from "swagger-schema-official" |
|||
const curlconverter = require("curlconverter") |
|||
import { URL } from "url" |
|||
|
|||
// {
|
|||
// "_id": "query_datasource_d62738f2d72a466997ffbf46f4952404_e7258ad382cd4c37961b81730633ff2d",
|
|||
// "_rev": "1-e702a18eaa96c7cb4be1b402c34eaa59",
|
|||
// "datasourceId": "datasource_d62738f2d72a466997ffbf46f4952404",
|
|||
// "parameters": [
|
|||
// {
|
|||
// "name": "paramtest",
|
|||
// "default": "defaultValue"
|
|||
// }
|
|||
// ],
|
|||
// "fields": {
|
|||
// "headers": {
|
|||
// "headertest": "test"
|
|||
// },
|
|||
// "queryString": "query=test",
|
|||
// "path": "/path/test"
|
|||
// },
|
|||
// "queryVerb": "read",
|
|||
// "transformer": "return data.test",
|
|||
// "schema": {},
|
|||
// "name": "name",
|
|||
// "readable": true
|
|||
// }
|
|||
|
|||
// return joiValidator.body(Joi.object({
|
|||
// _id: Joi.string(),
|
|||
// _rev: Joi.string(),
|
|||
// name: Joi.string().required(),
|
|||
// fields: Joi.object().required(),
|
|||
// datasourceId: Joi.string().required(),
|
|||
// readable: Joi.boolean(),
|
|||
// parameters: Joi.array().items(Joi.object({
|
|||
// name: Joi.string(),
|
|||
// default: Joi.string().allow(""),
|
|||
// })),
|
|||
// queryVerb: Joi.string().allow().required(),
|
|||
// extra: Joi.object().optional(),
|
|||
// schema: Joi.object({}).required().unknown(true),
|
|||
// transformer: Joi.string().optional(),
|
|||
// }))
|
|||
|
|||
interface Parameter { |
|||
name: string |
|||
default: string |
|||
} |
|||
|
|||
interface Query { |
|||
_id?: string |
|||
datasourceId: string |
|||
name: string |
|||
parameters: Parameter[] |
|||
fields: { |
|||
headers: any |
|||
queryString: string |
|||
path: string |
|||
} |
|||
transformer: string | null |
|||
schema: any |
|||
readable: boolean |
|||
queryVerb: string |
|||
} |
|||
|
|||
enum Strategy { |
|||
SWAGGER2, |
|||
OPENAPI3, |
|||
CURL, |
|||
} |
|||
|
|||
enum MethodToVerb { |
|||
get = "read", |
|||
post = "create", |
|||
put = "update", |
|||
patch = "patch", |
|||
delete = "delete", |
|||
} |
|||
|
|||
interface ImportResult { |
|||
errorQueries: Query[] |
|||
} |
|||
|
|||
interface DatasourceInfo { |
|||
url: string |
|||
name: string |
|||
defaultHeaders: any[] |
|||
} |
|||
|
|||
const parseImportStrategy = (data: string): Strategy => { |
|||
try { |
|||
const json = JSON.parse(data) |
|||
if (json.swagger === "2.0") { |
|||
return Strategy.CURL |
|||
} else if (json.openapi?.includes("3.0")) { |
|||
return Strategy.OPENAPI3 |
|||
} |
|||
} catch (jsonError) { |
|||
try { |
|||
parseCurl(data) |
|||
return Strategy.CURL |
|||
} catch (curlError) { |
|||
// do nothing
|
|||
} |
|||
} |
|||
|
|||
throw new Error(`The import data could not be processed`) |
|||
} |
|||
|
|||
const processPath = (path: string): string => { |
|||
if (path?.startsWith("/")) { |
|||
return path.substring(1) |
|||
} |
|||
|
|||
return path |
|||
} |
|||
|
|||
// SWAGGER
|
|||
|
|||
const parseSwagger2Info = (swagger2: Swagger2): DatasourceInfo => { |
|||
const scheme = swagger2.schemes?.includes("https") ? "https" : "http" |
|||
const basePath = swagger2.basePath || "" |
|||
const host = swagger2.host || "<host>" |
|||
const url = `${scheme}://${host}${basePath}` |
|||
const name = swagger2.info.title || "Swagger Import" |
|||
|
|||
return { |
|||
url: url, |
|||
name: name, |
|||
defaultHeaders: [], |
|||
} |
|||
} |
|||
|
|||
const parseSwagger2Queries = ( |
|||
datasourceId: string, |
|||
swagger2: Swagger2 |
|||
): Query[] => { |
|||
const queries = [] |
|||
|
|||
for (let [pathName, path] of Object.entries(swagger2.paths)) { |
|||
for (let [methodName, op] of Object.entries(path)) { |
|||
let operation = op as Operation |
|||
|
|||
const name = operation.operationId || pathName |
|||
const queryString = "" |
|||
const headers = {} |
|||
const parameters: Parameter[] = [] |
|||
|
|||
const query = constructQuery( |
|||
datasourceId, |
|||
name, |
|||
methodName, |
|||
pathName, |
|||
queryString, |
|||
headers, |
|||
parameters |
|||
) |
|||
queries.push(query) |
|||
} |
|||
} |
|||
|
|||
return queries |
|||
} |
|||
|
|||
// OPEN API
|
|||
|
|||
const parseOpenAPI3Info = (data: any): DatasourceInfo => { |
|||
return { |
|||
url: "http://localhost:3000", |
|||
name: "swagger", |
|||
defaultHeaders: [], |
|||
} |
|||
} |
|||
|
|||
const parseOpenAPI3Queries = (datasourceId: string, data: string): Query[] => { |
|||
return [] |
|||
} |
|||
|
|||
// CURL
|
|||
|
|||
const parseCurl = (data: string): any => { |
|||
const curlJson = curlconverter.toJsonString(data) |
|||
return JSON.parse(curlJson) |
|||
} |
|||
|
|||
const parseCurlDatasourceInfo = (data: any): DatasourceInfo => { |
|||
const curl = parseCurl(data) |
|||
|
|||
const url = new URL(curl.url) |
|||
|
|||
return { |
|||
url: url.origin, |
|||
name: url.hostname, |
|||
defaultHeaders: [], |
|||
} |
|||
} |
|||
|
|||
const parseCurlQueries = (datasourceId: string, data: string): Query[] => { |
|||
const curl = parseCurl(data) |
|||
|
|||
const url = new URL(curl.url) |
|||
const name = url.pathname |
|||
const path = url.pathname |
|||
const method = curl.method |
|||
const queryString = url.search |
|||
const headers = curl.headers |
|||
|
|||
const query = constructQuery( |
|||
datasourceId, |
|||
name, |
|||
method, |
|||
path, |
|||
queryString, |
|||
headers |
|||
) |
|||
return [query] |
|||
} |
|||
|
|||
const verbFromMethod = (method: string) => { |
|||
const verb = (<any>MethodToVerb)[method] |
|||
if (!verb) { |
|||
throw new Error(`Unsupported method: ${method}`) |
|||
} |
|||
return verb |
|||
} |
|||
|
|||
const constructQuery = ( |
|||
datasourceId: string, |
|||
name: string, |
|||
method: string, |
|||
path: string, |
|||
queryString: string, |
|||
headers: any = {}, |
|||
parameters: Parameter[] = [] |
|||
): Query => { |
|||
const readable = true |
|||
const queryVerb = verbFromMethod(method) |
|||
const transformer = "return data" |
|||
const schema = {} |
|||
path = processPath(path) |
|||
|
|||
const query: Query = { |
|||
datasourceId, |
|||
name, |
|||
parameters, |
|||
fields: { |
|||
headers, |
|||
queryString, |
|||
path, |
|||
}, |
|||
transformer, |
|||
schema, |
|||
readable, |
|||
queryVerb, |
|||
} |
|||
|
|||
return query |
|||
} |
|||
|
|||
export const getDatasourceInfo = (data: string): DatasourceInfo => { |
|||
const strategy = parseImportStrategy(data) |
|||
|
|||
let info: DatasourceInfo |
|||
switch (strategy) { |
|||
case Strategy.SWAGGER2: |
|||
info = parseSwagger2Info(JSON.parse(data)) |
|||
break |
|||
case Strategy.OPENAPI3: |
|||
info = parseOpenAPI3Info(JSON.parse(data)) |
|||
break |
|||
case Strategy.CURL: |
|||
info = parseCurlDatasourceInfo(data) |
|||
break |
|||
} |
|||
|
|||
return info |
|||
} |
|||
|
|||
export const importQueries = async ( |
|||
appId: string, |
|||
datasourceId: string, |
|||
data: string |
|||
): Promise<ImportResult> => { |
|||
const strategy = parseImportStrategy(data) |
|||
|
|||
// constuct the queries
|
|||
let queries: Query[] |
|||
switch (strategy) { |
|||
case Strategy.SWAGGER2: |
|||
queries = parseSwagger2Queries(datasourceId, JSON.parse(data)) |
|||
break |
|||
case Strategy.OPENAPI3: |
|||
queries = parseOpenAPI3Queries(datasourceId, JSON.parse(data)) |
|||
break |
|||
case Strategy.CURL: |
|||
queries = parseCurlQueries(datasourceId, data) |
|||
break |
|||
} |
|||
|
|||
// validate queries
|
|||
const errorQueries = [] |
|||
const schema = queryValidation() |
|||
queries = queries |
|||
.filter(query => { |
|||
const validation = schema.validate(query) |
|||
if (validation.error) { |
|||
errorQueries.push(query) |
|||
return false |
|||
} |
|||
return true |
|||
}) |
|||
.map(query => { |
|||
query._id = generateQueryID(query.datasourceId) |
|||
return query |
|||
}) |
|||
|
|||
// persist queries
|
|||
const db = new CouchDB(appId) |
|||
for (const query of queries) { |
|||
try { |
|||
await db.put(query) |
|||
} catch (error) { |
|||
errorQueries.push(query) |
|||
} |
|||
} |
|||
|
|||
return { |
|||
errorQueries, |
|||
} |
|||
} |
|||
@ -0,0 +1,78 @@ |
|||
import CouchDB from "../../../../db" |
|||
import { queryValidation } from "../validation" |
|||
import { generateQueryID } from "../../../../db/utils" |
|||
import { Query, ImportInfo, ImportSource } from "./sources/base" |
|||
import { OpenAPI2 } from "./sources/openapi2" |
|||
import { OpenAPI3 } from "./sources/openapi3" |
|||
import { Curl } from "./sources/curl" |
|||
|
|||
interface ImportResult { |
|||
errorQueries: Query[] |
|||
} |
|||
|
|||
export class RestImporter { |
|||
data: string |
|||
sources: ImportSource[] |
|||
source!: ImportSource |
|||
|
|||
constructor(data: string) { |
|||
this.data = data |
|||
this.sources = [new OpenAPI2(), new OpenAPI3(), new Curl()] |
|||
} |
|||
|
|||
init = async () => { |
|||
for (let source of this.sources) { |
|||
if (await source.isSupported(this.data)){ |
|||
this.source = source |
|||
break |
|||
} |
|||
} |
|||
} |
|||
|
|||
getInfo = async (): Promise<ImportInfo> => { |
|||
return this.source.getInfo() |
|||
} |
|||
|
|||
importQueries = async ( |
|||
appId: string, |
|||
datasourceId: string, |
|||
): Promise<ImportResult> => { |
|||
|
|||
// constuct the queries
|
|||
let queries = await this.source.getQueries(datasourceId) |
|||
|
|||
// validate queries
|
|||
const errorQueries = [] |
|||
const schema = queryValidation() |
|||
queries = queries |
|||
.filter(query => { |
|||
const validation = schema.validate(query) |
|||
if (validation.error) { |
|||
errorQueries.push(query) |
|||
return false |
|||
} |
|||
return true |
|||
}) |
|||
.map(query => { |
|||
query._id = generateQueryID(query.datasourceId) |
|||
return query |
|||
}) |
|||
|
|||
// persist queries
|
|||
const db = new CouchDB(appId) |
|||
for (const query of queries) { |
|||
try { |
|||
await db.put(query) |
|||
} catch (error) { |
|||
errorQueries.push(query) |
|||
} |
|||
} |
|||
|
|||
return { |
|||
errorQueries, |
|||
} |
|||
} |
|||
|
|||
} |
|||
|
|||
|
|||
@ -0,0 +1,92 @@ |
|||
export interface ImportInfo { |
|||
url: string |
|||
name: string |
|||
} |
|||
|
|||
export interface QueryParameter { |
|||
name: string |
|||
default: string |
|||
} |
|||
|
|||
export interface Query { |
|||
_id?: string |
|||
datasourceId: string |
|||
name: string |
|||
parameters: QueryParameter[] |
|||
fields: { |
|||
headers: object |
|||
queryString: string | null |
|||
path: string |
|||
requestBody?: object |
|||
} |
|||
transformer: string | null |
|||
schema: any |
|||
readable: boolean |
|||
queryVerb: string |
|||
} |
|||
|
|||
enum MethodToVerb { |
|||
get = "read", |
|||
post = "create", |
|||
put = "update", |
|||
patch = "patch", |
|||
delete = "delete", |
|||
} |
|||
|
|||
export abstract class ImportSource { |
|||
|
|||
abstract isSupported(data: string): Promise<boolean> |
|||
abstract getInfo(): Promise<ImportInfo> |
|||
abstract getQueries(datasourceId: string): Promise<Query[]> |
|||
|
|||
constructQuery = ( |
|||
datasourceId: string, |
|||
name: string, |
|||
method: string, |
|||
path: string, |
|||
queryString: string, |
|||
headers: object = {}, |
|||
parameters: QueryParameter[] = [], |
|||
requestBody: object | undefined = undefined, |
|||
): Query => { |
|||
const readable = true |
|||
const queryVerb = this.verbFromMethod(method) |
|||
const transformer = "return data" |
|||
const schema = {} |
|||
path = this.processPath(path) |
|||
|
|||
const query: Query = { |
|||
datasourceId, |
|||
name, |
|||
parameters, |
|||
fields: { |
|||
headers, |
|||
queryString, |
|||
path, |
|||
requestBody |
|||
}, |
|||
transformer, |
|||
schema, |
|||
readable, |
|||
queryVerb, |
|||
} |
|||
|
|||
return query |
|||
} |
|||
|
|||
verbFromMethod = (method: string) => { |
|||
const verb = (<any>MethodToVerb)[method] |
|||
if (!verb) { |
|||
throw new Error(`Unsupported method: ${method}`) |
|||
} |
|||
return verb |
|||
} |
|||
|
|||
processPath = (path: string): string => { |
|||
if (path?.startsWith("/")) { |
|||
return path.substring(1) |
|||
} |
|||
|
|||
return path |
|||
} |
|||
} |
|||
@ -0,0 +1,13 @@ |
|||
|
|||
import { ImportSource } from "." |
|||
import SwaggerParser from "@apidevtools/swagger-parser"; |
|||
import { OpenAPI } from "openapi-types"; |
|||
|
|||
export abstract class OpenAPISource extends ImportSource { |
|||
|
|||
parseData = async (data: string): Promise<OpenAPI.Document> => { |
|||
const json = JSON.parse(data) |
|||
return SwaggerParser.validate(json, {}) |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,54 @@ |
|||
import { ImportSource, ImportInfo, Query } from "./base" |
|||
import { URL } from 'url' |
|||
const curlconverter = require("curlconverter") |
|||
|
|||
const parseCurl = (data: string): any => { |
|||
const curlJson = curlconverter.toJsonString(data) |
|||
return JSON.parse(curlJson) |
|||
} |
|||
|
|||
/** |
|||
* Curl |
|||
* https://curl.se/docs/manpage.html
|
|||
*/ |
|||
export class Curl extends ImportSource { |
|||
curl: any |
|||
|
|||
isSupported = async (data: string): Promise<boolean> => { |
|||
try { |
|||
const curl = parseCurl(data) |
|||
this.curl = curl |
|||
} catch (err) { |
|||
return false |
|||
} |
|||
return true |
|||
} |
|||
|
|||
getInfo = async (): Promise<ImportInfo> => { |
|||
const url = new URL(this.curl.url) |
|||
return { |
|||
url: url.origin, |
|||
name: url.hostname, |
|||
} |
|||
} |
|||
|
|||
getQueries = async (datasourceId: string): Promise<Query[]> => { |
|||
const url = new URL(this.curl.url) |
|||
const name = url.pathname |
|||
const path = url.pathname |
|||
const method = this.curl.method |
|||
const queryString = url.search |
|||
const headers = this.curl.headers |
|||
|
|||
const query = this.constructQuery( |
|||
datasourceId, |
|||
name, |
|||
method, |
|||
path, |
|||
queryString, |
|||
headers |
|||
) |
|||
|
|||
return [query] |
|||
} |
|||
} |
|||
@ -0,0 +1,104 @@ |
|||
import { ImportInfo, QueryParameter, Query } from "./base" |
|||
import { OpenAPIV2 } from "openapi-types" |
|||
import { OpenAPISource } from "./base/openapi"; |
|||
|
|||
const isBodyParameter = (param: OpenAPIV2.Parameter): param is OpenAPIV2.InBodyParameterObject => { |
|||
return param.in === "body" |
|||
} |
|||
|
|||
const isParameter = (param: OpenAPIV2.Parameter | OpenAPIV2.ReferenceObject): param is OpenAPIV2.Parameter => { |
|||
// we can guarantee this is not a reference object
|
|||
// due to the deferencing done by the parser library
|
|||
return true |
|||
} |
|||
|
|||
const isOpenAPI2 = (document: any): document is OpenAPIV2.Document => { |
|||
if (document.swagger === "2.0") { |
|||
return true |
|||
} else { |
|||
return false |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* OpenAPI Version 2.0 - aka "Swagger" |
|||
* https://github.com/OAI/OpenAPI-Specification/blob/main/versions/2.0.md
|
|||
*/ |
|||
export class OpenAPI2 extends OpenAPISource { |
|||
document!: OpenAPIV2.Document |
|||
|
|||
isSupported = async (data: string): Promise<boolean> => { |
|||
try { |
|||
const document: any = await this.parseData(data) |
|||
if (isOpenAPI2(document)) { |
|||
this.document = document |
|||
return true |
|||
} else { |
|||
return false |
|||
} |
|||
} catch (err) { |
|||
return false |
|||
} |
|||
} |
|||
|
|||
getInfo = async (): Promise<ImportInfo> => { |
|||
const scheme = this.document.schemes?.includes("https") ? "https" : "http" |
|||
const basePath = this.document.basePath || "" |
|||
const host = this.document.host || "<host>" |
|||
const url = `${scheme}://${host}${basePath}` |
|||
const name = this.document.info.title || "Swagger Import" |
|||
|
|||
return { |
|||
url: url, |
|||
name: name, |
|||
} |
|||
} |
|||
|
|||
getQueries = async (datasourceId: string): Promise<Query[]> => { |
|||
const queries = [] |
|||
|
|||
let pathName: string |
|||
let path: OpenAPIV2.PathItemObject |
|||
|
|||
for ([pathName, path] of Object.entries(this.document.paths)) { |
|||
for (let [methodName, op] of Object.entries(path)) { |
|||
let operation = op as OpenAPIV2.OperationObject |
|||
|
|||
const name = operation.operationId || pathName |
|||
const queryString = "" |
|||
const headers = {} |
|||
let requestBody = undefined |
|||
const parameters: QueryParameter[] = [] |
|||
|
|||
if (operation.parameters) { |
|||
for (let param of operation.parameters) { |
|||
if (isParameter(param)) { |
|||
if (isBodyParameter(param)) { |
|||
requestBody = {} |
|||
} else { |
|||
parameters.push({ |
|||
name: param.name, |
|||
default: "", |
|||
}) |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
const query = this.constructQuery( |
|||
datasourceId, |
|||
name, |
|||
methodName, |
|||
pathName, |
|||
queryString, |
|||
headers, |
|||
parameters, |
|||
requestBody |
|||
) |
|||
queries.push(query) |
|||
} |
|||
} |
|||
|
|||
return queries |
|||
} |
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
import { ImportInfo, Query } from "./base" |
|||
import { OpenAPISource } from "./base/openapi" |
|||
import { OpenAPIV3 } from "openapi-types" |
|||
|
|||
const isOpenAPI3 = (document: any): document is OpenAPIV3.Document => { |
|||
return document.openapi === "3.0.0" |
|||
} |
|||
|
|||
/** |
|||
* OpenAPI Version 3.0.0 |
|||
* https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.0.md
|
|||
*/ |
|||
export class OpenAPI3 extends OpenAPISource { |
|||
document!: OpenAPIV3.Document |
|||
|
|||
isSupported = async (data: string): Promise<boolean> => { |
|||
try { |
|||
const document: any = await this.parseData(data) |
|||
if (isOpenAPI3(document)) { |
|||
this.document = document |
|||
return true |
|||
} else { |
|||
return false |
|||
} |
|||
} catch (err) { |
|||
return false |
|||
} |
|||
} |
|||
|
|||
getInfo = async (): Promise<ImportInfo> => { |
|||
return { |
|||
url: "http://localhost:3000", |
|||
name: "swagger", |
|||
} |
|||
} |
|||
|
|||
getQueries = async (datasourceId: string): Promise<Query[]> => { |
|||
return [] |
|||
} |
|||
} |
|||
@ -0,0 +1,71 @@ |
|||
// const Airtable = require("airtable")
|
|||
// const AirtableIntegration = require("../airtable")
|
|||
|
|||
jest.mock("airtable") |
|||
|
|||
// class TestConfiguration {
|
|||
// constructor(config = {}) {
|
|||
// this.integration = new AirtableIntegration.integration(config)
|
|||
// this.client = {
|
|||
// create: jest.fn(),
|
|||
// select: jest.fn(),
|
|||
// update: jest.fn(),
|
|||
// destroy: jest.fn(),
|
|||
// }
|
|||
// this.integration.client = () => this.client
|
|||
// }
|
|||
// }
|
|||
|
|||
describe("Airtable Integration", () => { |
|||
let config |
|||
|
|||
beforeEach(() => { |
|||
config = new TestConfiguration() |
|||
}) |
|||
|
|||
it("calls the create method with the correct params", async () => { |
|||
const response = await config.integration.create({ |
|||
table: "test", |
|||
json: {} |
|||
}) |
|||
expect(config.client.create).toHaveBeenCalledWith([ |
|||
{ |
|||
fields: {} |
|||
} |
|||
]) |
|||
}) |
|||
|
|||
it("calls the read method with the correct params", async () => { |
|||
const response = await config.integration.read({ |
|||
table: "test", |
|||
view: "Grid view" |
|||
}) |
|||
expect(config.client.select).toHaveBeenCalledWith({ |
|||
maxRecords: 10, view: "Grid view" |
|||
}) |
|||
}) |
|||
|
|||
it("calls the update method with the correct params", async () => { |
|||
const response = await config.integration.update({ |
|||
table: "test", |
|||
id: "123", |
|||
json: { |
|||
name: "test" |
|||
} |
|||
}) |
|||
expect(config.client.update).toHaveBeenCalledWith([ |
|||
{ |
|||
id: "123", |
|||
fields: { name: "test" } |
|||
} |
|||
]) |
|||
}) |
|||
|
|||
it("calls the delete method with the correct params", async () => { |
|||
const ids = [1,2,3,4] |
|||
const response = await config.integration.delete({ |
|||
ids |
|||
}) |
|||
expect(config.client.destroy).toHaveBeenCalledWith(ids) |
|||
}) |
|||
}) |
|||
Loading…
Reference in new issue